[
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000059241894633,
    "completed": 295,
    "accepted": 295,
    "sent": 299,
    "late": 0,
    "completed_fps": 9.833313915195104,
    "accepted_fps": 9.833313915195104,
    "sent_fps": 9.966646985231648,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.792611649,
    "selected": 295,
    "infer_entries": 294,
    "infer_entry_fps": 9.86821845173376,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "copy_wall": {
        "count": 295,
        "sum_ns": 2951640953,
        "mean_ms": 10.005562552542374,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          58,
          194,
          38,
          2,
          0,
          0,
          0,
          0,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 308.420433
      },
      "copy_thread_cpu": {
        "count": 295,
        "sum_ns": 2846719585,
        "mean_ms": 9.649896898305084,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          63,
          201,
          26,
          2,
          0,
          0,
          0,
          0,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 301.500596
      },
      "accepted_interval": {
        "count": 295,
        "sum_ns": 29445293797,
        "mean_ms": 99.8145552440678,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          4,
          11,
          13,
          17,
          113,
          126,
          2,
          0,
          9,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 430.95454
      },
      "completed_queue_wait": {
        "count": 295,
        "sum_ns": 1144347940,
        "mean_ms": 3.879145559322034,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          212,
          30,
          26,
          1,
          4,
          15,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 60.155267
      },
      "selected_to_preprocess_start": {
        "count": 294,
        "sum_ns": 788990404,
        "mean_ms": 2.683640829931973,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          119,
          150,
          22,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.932553
      },
      "preprocess_start_to_preprocess_output": {
        "count": 294,
        "sum_ns": 558871591,
        "mean_ms": 1.9009237789115647,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          105,
          117,
          55,
          13,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 16.437965
      },
      "selected_to_preprocess": {
        "count": 294,
        "sum_ns": 1663629945,
        "mean_ms": 5.658605255102041,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          189,
          73,
          26,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.41587
      },
      "preprocess_start_to_preprocess": {
        "count": 294,
        "sum_ns": 874639541,
        "mean_ms": 2.974964425170068,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          3,
          158,
          95,
          27,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.586426
      },
      "preprocess_output_to_infer_entry": {
        "count": 294,
        "sum_ns": 315767950,
        "mean_ms": 1.0740406462585035,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          234,
          25,
          22,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 13.18028
      },
      "preprocess_to_infer_output": {
        "count": 294,
        "sum_ns": 10882736196,
        "mean_ms": 37.01610951020408,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          271,
          23,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.154548
      },
      "preprocess_to_callback": {
        "count": 294,
        "sum_ns": 11557424880,
        "mean_ms": 39.31096897959184,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          268,
          24,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 83.84466
      },
      "infer_output_to_callback": {
        "count": 294,
        "sum_ns": 674688684,
        "mean_ms": 2.294859469387755,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          4,
          162,
          111,
          15,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.936478
      },
      "callback_to_copied": {
        "count": 294,
        "sum_ns": 3132752872,
        "mean_ms": 10.655622013605441,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          35,
          198,
          55,
          3,
          0,
          0,
          0,
          0,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 309.307591
      },
      "selected_to_submitted": {
        "count": 294,
        "sum_ns": 16466912284,
        "mean_ms": 56.00990572789115,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          130,
          145,
          16,
          0,
          0,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 356.958271
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.37550678802654,
    "first_redis_commit_s": 21.44207639945671,
    "maximum_sampled_silence_ms": 348.604055,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000059241894633,
    "completed": 298,
    "accepted": 298,
    "sent": 299,
    "late": 0,
    "completed_fps": 9.933313717722513,
    "accepted_fps": 9.933313717722513,
    "sent_fps": 9.966646985231648,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.778377801,
    "selected": 297,
    "infer_entries": 298,
    "infer_entry_fps": 10.007261039921145,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "copy_wall": {
        "count": 298,
        "sum_ns": 4985791406,
        "mean_ms": 16.7308436442953,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          231,
          67,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.120523
      },
      "copy_thread_cpu": {
        "count": 298,
        "sum_ns": 4803522246,
        "mean_ms": 16.119202167785236,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          248,
          50,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.386522
      },
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29766011349,
        "mean_ms": 99.88594412416109,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          26,
          112,
          155,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 166.367919
      },
      "completed_queue_wait": {
        "count": 298,
        "sum_ns": 223480608,
        "mean_ms": 0.7499349261744966,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          245,
          35,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 3.481755
      },
      "selected_to_preprocess_start": {
        "count": 297,
        "sum_ns": 665393010,
        "mean_ms": 2.240380505050505,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          0,
          144,
          146,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 9.279735
      },
      "preprocess_start_to_preprocess_output": {
        "count": 297,
        "sum_ns": 445875060,
        "mean_ms": 1.5012628282828282,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          114,
          134,
          41,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 6.534806
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 1255426803,
        "mean_ms": 4.227026272727272,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          0,
          243,
          50,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 13.887626
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 590033793,
        "mean_ms": 1.9866457676767677,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          6,
          196,
          83,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.147674
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 144158733,
        "mean_ms": 0.4853829393939394,
        "p95_bucket_upper_ms": 1,
        "histogram": [
          292,
          3,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 5.894955
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 7991589938,
        "mean_ms": 26.90771022895623,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          16,
          281,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.190476
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 8741822705,
        "mean_ms": 29.43374648148148,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          286,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.075806
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 750232767,
        "mean_ms": 2.5260362525252527,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          0,
          113,
          177,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 6.519399
      },
      "callback_to_copied": {
        "count": 297,
        "sum_ns": 5189825259,
        "mean_ms": 17.47415912121212,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          215,
          82,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.150677
      },
      "selected_to_submitted": {
        "count": 297,
        "sum_ns": 15290112301,
        "mean_ms": 51.4818595993266,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          132,
          163,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.336232
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.20394528610632,
    "first_redis_commit_s": 21.238167728297412,
    "maximum_sampled_silence_ms": 108.57287,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000059241894633,
    "completed": 300,
    "accepted": 300,
    "sent": 300,
    "late": 0,
    "completed_fps": 9.999980252740786,
    "accepted_fps": 9.999980252740786,
    "sent_fps": 9.999980252740786,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.768684153,
    "selected": 301,
    "infer_entries": 297,
    "infer_entry_fps": 9.976927380247314,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29726947819,
        "mean_ms": 99.75485845302013,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          3,
          24,
          118,
          150,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 159.76647
      },
      "completed_queue_wait": {
        "count": 298,
        "sum_ns": 183681994,
        "mean_ms": 0.6163825302013423,
        "p95_bucket_upper_ms": 2,
        "histogram": [
          270,
          23,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 4.538886
      },
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 670498159,
        "mean_ms": 2.249993822147651,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          0,
          133,
          159,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 8.233954
      },
      "preprocess_start_to_preprocess_output": {
        "count": 297,
        "sum_ns": 421965033,
        "mean_ms": 1.4207576868686869,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          110,
          145,
          35,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 6.49784
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 1237653988,
        "mean_ms": 4.167185144781145,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          0,
          236,
          58,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.156723
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 569339376,
        "mean_ms": 1.9169675959595958,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          7,
          206,
          76,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.04505
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 147374343,
        "mean_ms": 0.4962099090909091,
        "p95_bucket_upper_ms": 1,
        "histogram": [
          288,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 3.557845
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 7894704192,
        "mean_ms": 26.581495595959595,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          22,
          273,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 51.466971
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 8678503006,
        "mean_ms": 29.22054884175084,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          285,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.26167
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 783798814,
        "mean_ms": 2.6390532457912457,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          96,
          186,
          15,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 7.216086
      },
      "copy_wall": {
        "count": 297,
        "sum_ns": 5072686196,
        "mean_ms": 17.079751501683504,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          228,
          69,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.054132
      },
      "copy_thread_cpu": {
        "count": 297,
        "sum_ns": 4947527105,
        "mean_ms": 16.658340420875422,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          241,
          56,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.399443
      },
      "callback_to_copied": {
        "count": 297,
        "sum_ns": 5306077789,
        "mean_ms": 17.865581781144783,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          219,
          78,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.094689
      },
      "selected_to_submitted": {
        "count": 297,
        "sum_ns": 15335511080,
        "mean_ms": 51.63471744107744,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          123,
          172,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 80.968877
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.152840240392834,
    "first_redis_commit_s": 22.59020359022543,
    "maximum_sampled_silence_ms": 67.997188,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.999972133897245,
    "completed": 297,
    "accepted": 297,
    "sent": 294,
    "late": 0,
    "completed_fps": 9.90000919582245,
    "accepted_fps": 9.90000919582245,
    "sent_fps": 9.800009102935356,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.766923558,
    "selected": 298,
    "infer_entries": 298,
    "infer_entry_fps": 10.011111810710151,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "copy_wall": {
        "count": 297,
        "sum_ns": 4466445391,
        "mean_ms": 15.038536670033672,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          2,
          100,
          132,
          60,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.857339
      },
      "copy_thread_cpu": {
        "count": 297,
        "sum_ns": 3313365359,
        "mean_ms": 11.15611231986532,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          3,
          139,
          143,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.349634
      },
      "accepted_interval": {
        "count": 297,
        "sum_ns": 29735338494,
        "mean_ms": 100.11898482828283,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          37,
          48,
          81,
          113,
          7,
          1,
          9,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 431.766115
      },
      "completed_queue_wait": {
        "count": 297,
        "sum_ns": 3376378888,
        "mean_ms": 11.368279084175084,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          139,
          48,
          46,
          18,
          9,
          13,
          8,
          8,
          5,
          1,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 244.675042
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 1584403580,
        "mean_ms": 5.352714797297297,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          95,
          125,
          45,
          25,
          5,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 269.206249
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 2938346746,
        "mean_ms": 9.926847114864865,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          28,
          71,
          70,
          73,
          38,
          11,
          1,
          0,
          0,
          0,
          2,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 353.757342
      },
      "selected_to_preprocess": {
        "count": 298,
        "sum_ns": 5838560540,
        "mean_ms": 19.592485033557047,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          75,
          72,
          79,
          62,
          2,
          2,
          0,
          0,
          3,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 368.968875
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 4212040212,
        "mean_ms": 14.22986558108108,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          63,
          63,
          68,
          63,
          33,
          0,
          1,
          0,
          0,
          2,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 354.537796
      },
      "preprocess_output_to_infer_entry": {
        "count": 298,
        "sum_ns": 1281341031,
        "mean_ms": 4.299802117449665,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          147,
          35,
          45,
          40,
          28,
          2,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 285.744801
      },
      "preprocess_to_infer_output": {
        "count": 296,
        "sum_ns": 18331293033,
        "mean_ms": 61.930044030405405,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          125,
          110,
          33,
          22,
          3,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 318.587505
      },
      "preprocess_to_callback": {
        "count": 296,
        "sum_ns": 19602693151,
        "mean_ms": 66.22531469932433,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          102,
          116,
          45,
          26,
          4,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 335.316093
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 1271400118,
        "mean_ms": 4.295270668918919,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          83,
          141,
          44,
          23,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.498099
      },
      "callback_to_copied": {
        "count": 297,
        "sum_ns": 4719099686,
        "mean_ms": 15.889224531986534,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          80,
          145,
          68,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.531123
      },
      "selected_to_submitted": {
        "count": 297,
        "sum_ns": 30466897991,
        "mean_ms": 102.58214811784512,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          9,
          88,
          83,
          87,
          14,
          9,
          7,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 431.487582
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.61393053177744,
    "first_redis_commit_s": 18.744325953070074,
    "maximum_sampled_silence_ms": 116.20383,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.999972133897245,
    "completed": 299,
    "accepted": 299,
    "sent": 298,
    "late": 0,
    "completed_fps": 9.966675924413847,
    "accepted_fps": 9.966675924413847,
    "sent_fps": 9.933342560118149,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.00535491,
    "selected": 300,
    "infer_entries": 300,
    "infer_entry_fps": 9.998215348554929,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 300,
        "sum_ns": 978941961,
        "mean_ms": 3.2631398700000003,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          112,
          145,
          32,
          8,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.838179
      },
      "preprocess_start_to_preprocess_output": {
        "count": 300,
        "sum_ns": 966286639,
        "mean_ms": 3.2209554633333335,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          53,
          104,
          81,
          48,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.958353
      },
      "selected_to_preprocess": {
        "count": 300,
        "sum_ns": 2377571664,
        "mean_ms": 7.92523888,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          133,
          102,
          48,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.943913
      },
      "preprocess_start_to_preprocess": {
        "count": 300,
        "sum_ns": 1398629703,
        "mean_ms": 4.6620990099999995,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          3,
          101,
          109,
          57,
          23,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 38.09582
      },
      "preprocess_output_to_infer_entry": {
        "count": 300,
        "sum_ns": 432343064,
        "mean_ms": 1.4411435466666667,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          231,
          25,
          20,
          17,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.137467
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 12717563619,
        "mean_ms": 42.53365758862876,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          197,
          81,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 96.56606
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 13769990587,
        "mean_ms": 46.05348022408027,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          190,
          88,
          18,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 107.137735
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 1052426968,
        "mean_ms": 3.5198226354515048,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          51,
          210,
          31,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.375655
      },
      "copy_wall": {
        "count": 299,
        "sum_ns": 7182149350,
        "mean_ms": 24.020566387959867,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          93,
          204,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.127736
      },
      "copy_thread_cpu": {
        "count": 299,
        "sum_ns": 5631001651,
        "mean_ms": 18.832781441471575,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          199,
          100,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.473031
      },
      "callback_to_copied": {
        "count": 299,
        "sum_ns": 7463872911,
        "mean_ms": 24.962785655518395,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          71,
          225,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.867264
      },
      "accepted_interval": {
        "count": 299,
        "sum_ns": 29928035792,
        "mean_ms": 100.09376519063545,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          11,
          34,
          112,
          131,
          10,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 211.898065
      },
      "selected_to_submitted": {
        "count": 299,
        "sum_ns": 23764338411,
        "mean_ms": 79.47939267892976,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          11,
          139,
          98,
          49,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 165.489015
      },
      "completed_queue_wait": {
        "count": 299,
        "sum_ns": 477095106,
        "mean_ms": 1.5956358060200668,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          166,
          58,
          60,
          12,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 16.997945
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.428674277849495,
    "first_redis_commit_s": 18.527694292832166,
    "maximum_sampled_silence_ms": 92.391293,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.999972133897245,
    "completed": 299,
    "accepted": 299,
    "sent": 297,
    "late": 0,
    "completed_fps": 9.966675924413847,
    "accepted_fps": 9.966675924413847,
    "sent_fps": 9.90000919582245,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.684357826,
    "selected": 299,
    "infer_entries": 297,
    "infer_entry_fps": 10.00526950055369,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 297,
        "sum_ns": 902822143,
        "mean_ms": 3.039805195286195,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          109,
          154,
          24,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.463316
      },
      "preprocess_start_to_preprocess_output": {
        "count": 297,
        "sum_ns": 919907154,
        "mean_ms": 3.097330484848485,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          52,
          107,
          80,
          40,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.921628
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 2236072545,
        "mean_ms": 7.528863787878788,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          144,
          99,
          34,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 37.192717
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 1333250402,
        "mean_ms": 4.489058592592593,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          4,
          110,
          103,
          44,
          29,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.93291
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 413343248,
        "mean_ms": 1.3917281077441077,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          233,
          20,
          19,
          21,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.233659
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 12940879645,
        "mean_ms": 43.57198533670034,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          190,
          76,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 130.728755
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 13963226243,
        "mean_ms": 47.01422977441078,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          182,
          87,
          22,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 136.053929
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 1022346598,
        "mean_ms": 3.4422444377104378,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          31,
          224,
          40,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.173235
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 6620083834,
        "mean_ms": 22.365148087837838,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          130,
          166,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.646135
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5376576646,
        "mean_ms": 18.16411029054054,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          221,
          75,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.495032
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 6888441845,
        "mean_ms": 23.271762989864865,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          106,
          190,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.461829
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29618073341,
        "mean_ms": 100.06105858445946,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          11,
          48,
          94,
          128,
          14,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 203.462049
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 23162702646,
        "mean_ms": 78.25237380405405,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          19,
          131,
          98,
          44,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 164.342219
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 332603761,
        "mean_ms": 1.1236613547297296,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          207,
          49,
          38,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.867391
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.481112030800432,
    "first_redis_commit_s": 20.354287057649344,
    "maximum_sampled_silence_ms": 95.082653,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 29.999972133897245,
    "completed": 302,
    "accepted": 302,
    "sent": 297,
    "late": 0,
    "completed_fps": 10.066676017300944,
    "accepted_fps": 10.066676017300944,
    "sent_fps": 9.90000919582245,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.257127483,
    "selected": 302,
    "infer_entries": 303,
    "infer_entry_fps": 10.01416939431018,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "copy_wall": {
        "count": 302,
        "sum_ns": 7691690800,
        "mean_ms": 25.469174834437087,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          35,
          116,
          132,
          15,
          2,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 474.359619
      },
      "copy_thread_cpu": {
        "count": 302,
        "sum_ns": 5246336163,
        "mean_ms": 17.371974049668875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          61,
          193,
          46,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 441.408877
      },
      "accepted_interval": {
        "count": 302,
        "sum_ns": 30196315888,
        "mean_ms": 99.98780095364238,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          11,
          60,
          66,
          53,
          87,
          14,
          0,
          1,
          10,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 648.729065
      },
      "completed_queue_wait": {
        "count": 302,
        "sum_ns": 42770969898,
        "mean_ms": 141.62572813907283,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          45,
          28,
          35,
          4,
          5,
          23,
          16,
          12,
          23,
          27,
          27,
          46,
          11,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 857.539714
      },
      "selected_to_preprocess_start": {
        "count": 300,
        "sum_ns": 2319627116,
        "mean_ms": 7.732090386666667,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          71,
          104,
          71,
          44,
          9,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 497.198167
      },
      "preprocess_start_to_preprocess_output": {
        "count": 298,
        "sum_ns": 2775017622,
        "mean_ms": 9.312139671140939,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          10,
          46,
          66,
          100,
          54,
          21,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 473.57543
      },
      "selected_to_preprocess": {
        "count": 303,
        "sum_ns": 6832509577,
        "mean_ms": 22.549536557755776,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          28,
          52,
          103,
          109,
          8,
          1,
          0,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 503.181771
      },
      "preprocess_start_to_preprocess": {
        "count": 301,
        "sum_ns": 4413361597,
        "mean_ms": 14.66233088704319,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          24,
          48,
          85,
          79,
          61,
          2,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 477.28854
      },
      "preprocess_output_to_infer_entry": {
        "count": 301,
        "sum_ns": 1613404234,
        "mean_ms": 5.360146956810631,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          97,
          28,
          62,
          66,
          35,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.59375
      },
      "preprocess_to_infer_output": {
        "count": 302,
        "sum_ns": 22926706276,
        "mean_ms": 75.91624594701986,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          92,
          102,
          47,
          50,
          5,
          3,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 595.920157
      },
      "preprocess_to_callback": {
        "count": 302,
        "sum_ns": 24933916539,
        "mean_ms": 82.56263754635762,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          69,
          101,
          58,
          62,
          5,
          4,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 623.971841
      },
      "infer_output_to_callback": {
        "count": 302,
        "sum_ns": 2007210263,
        "mean_ms": 6.646391599337748,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          35,
          110,
          98,
          51,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.912979
      },
      "callback_to_copied": {
        "count": 302,
        "sum_ns": 7921801453,
        "mean_ms": 26.23113063907285,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          30,
          117,
          135,
          16,
          2,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 475.470786
      },
      "selected_to_submitted": {
        "count": 302,
        "sum_ns": 40316676222,
        "mean_ms": 133.4989278874172,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          38,
          69,
          110,
          58,
          19,
          2,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 669.566685
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.37550678802654,
    "first_redis_commit_s": 21.44207639945671,
    "maximum_sampled_silence_ms": 459.056684,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 29.999972133897245,
    "completed": 300,
    "accepted": 300,
    "sent": 297,
    "late": 0,
    "completed_fps": 10.000009288709546,
    "accepted_fps": 10.000009288709546,
    "sent_fps": 9.90000919582245,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.01464091,
    "selected": 301,
    "infer_entries": 300,
    "infer_entry_fps": 9.995122077240936,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 300,
        "sum_ns": 6942969510,
        "mean_ms": 23.143231699999998,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          116,
          181,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.594718
      },
      "copy_thread_cpu": {
        "count": 300,
        "sum_ns": 5658125918,
        "mean_ms": 18.860419726666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          200,
          100,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.207595
      },
      "accepted_interval": {
        "count": 300,
        "sum_ns": 30039702410,
        "mean_ms": 100.13234136666667,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          14,
          48,
          98,
          120,
          20,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 198.089107
      },
      "completed_queue_wait": {
        "count": 300,
        "sum_ns": 369211979,
        "mean_ms": 1.2307065966666666,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          198,
          53,
          43,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.596573
      },
      "selected_to_preprocess_start": {
        "count": 301,
        "sum_ns": 1102603116,
        "mean_ms": 3.663133275747508,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          103,
          141,
          41,
          12,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 26.127531
      },
      "preprocess_start_to_preprocess_output": {
        "count": 301,
        "sum_ns": 946960650,
        "mean_ms": 3.1460486710963456,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          51,
          103,
          89,
          45,
          13,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.021286
      },
      "selected_to_preprocess": {
        "count": 300,
        "sum_ns": 2426552248,
        "mean_ms": 8.088507493333333,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          139,
          92,
          44,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.000012
      },
      "preprocess_start_to_preprocess": {
        "count": 300,
        "sum_ns": 1334410138,
        "mean_ms": 4.448033793333333,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          5,
          107,
          108,
          49,
          26,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.333635
      },
      "preprocess_output_to_infer_entry": {
        "count": 300,
        "sum_ns": 394162234,
        "mean_ms": 1.3138741133333334,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          231,
          22,
          26,
          16,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.184971
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 11920745277,
        "mean_ms": 39.868713301003346,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          19,
          205,
          62,
          13,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 97.232679
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 13035316805,
        "mean_ms": 43.59637727424749,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          194,
          81,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 99.717421
      },
      "infer_output_to_callback": {
        "count": 300,
        "sum_ns": 1117087875,
        "mean_ms": 3.72362625,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          35,
          210,
          53,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 11.277255
      },
      "callback_to_copied": {
        "count": 300,
        "sum_ns": 7256774971,
        "mean_ms": 24.189249903333334,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          95,
          201,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 56.7039
      },
      "selected_to_submitted": {
        "count": 300,
        "sum_ns": 22939523583,
        "mean_ms": 76.46507861,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          28,
          133,
          92,
          46,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.133624
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.20394528610632,
    "first_redis_commit_s": 21.238167728297412,
    "maximum_sampled_silence_ms": 119.609318,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 29.999972133897245,
    "completed": 300,
    "accepted": 300,
    "sent": 295,
    "late": 0,
    "completed_fps": 10.000009288709546,
    "accepted_fps": 10.000009288709546,
    "sent_fps": 9.833342467231054,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.940041963,
    "selected": 299,
    "infer_entries": 299,
    "infer_entry_fps": 9.986625949606388,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "copy_wall": {
        "count": 300,
        "sum_ns": 6803337771,
        "mean_ms": 22.67779257,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          135,
          162,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.538195
      },
      "copy_thread_cpu": {
        "count": 300,
        "sum_ns": 5637257672,
        "mean_ms": 18.790858906666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          197,
          103,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.013353
      },
      "accepted_interval": {
        "count": 300,
        "sum_ns": 29993646385,
        "mean_ms": 99.97882128333333,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          12,
          58,
          91,
          117,
          22,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 194.0034
      },
      "completed_queue_wait": {
        "count": 300,
        "sum_ns": 376045880,
        "mean_ms": 1.2534862666666666,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          192,
          60,
          44,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 9.717414
      },
      "selected_to_preprocess_start": {
        "count": 297,
        "sum_ns": 921464712,
        "mean_ms": 3.102574787878788,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          99,
          159,
          29,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.918846
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 844840279,
        "mean_ms": 2.854190131756757,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          60,
          109,
          74,
          42,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.303441
      },
      "selected_to_preprocess": {
        "count": 298,
        "sum_ns": 2115493599,
        "mean_ms": 7.098971808724832,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          1,
          150,
          92,
          42,
          13,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.996712
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 1175892184,
        "mean_ms": 3.959232942760943,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          4,
          130,
          88,
          52,
          20,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 26.813819
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 310075247,
        "mean_ms": 1.0440244006734007,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          243,
          22,
          19,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.984095
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 11739718216,
        "mean_ms": 39.26327162541806,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          21,
          209,
          53,
          15,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 130.46959
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 12867506076,
        "mean_ms": 43.035137377926425,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          8,
          196,
          77,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 132.446003
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 1127787860,
        "mean_ms": 3.7718657525083614,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          31,
          215,
          47,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.150058
      },
      "callback_to_copied": {
        "count": 299,
        "sum_ns": 7095945579,
        "mean_ms": 23.73225946153846,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          109,
          187,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.155571
      },
      "selected_to_submitted": {
        "count": 299,
        "sum_ns": 22265585967,
        "mean_ms": 74.46684269899664,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          32,
          135,
          92,
          39,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 153.05646
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.152840240392834,
    "first_redis_commit_s": 22.59020359022543,
    "maximum_sampled_silence_ms": 140.709949,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 188,
    "accepted": 188,
    "sent": 158,
    "late": 0,
    "completed_fps": 6.266181272654544,
    "accepted_fps": 6.266181272654544,
    "sent_fps": 5.26625872914584,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.396171025,
    "selected": 188,
    "infer_entries": 189,
    "infer_entry_fps": 6.429408777056875,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 186,
        "sum_ns": 2043417938,
        "mean_ms": 10.986117946236558,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          45,
          55,
          16,
          37,
          24,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 69.126251
      },
      "copy_wall": {
        "count": 188,
        "sum_ns": 10474545147,
        "mean_ms": 55.715665675531916,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          27,
          80,
          47,
          13,
          11,
          1,
          0,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 591.368002
      },
      "copy_thread_cpu": {
        "count": 188,
        "sum_ns": 3898499047,
        "mean_ms": 20.73669705851064,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          18,
          130,
          37,
          0,
          0,
          0,
          0,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 342.810322
      },
      "accepted_interval": {
        "count": 188,
        "sum_ns": 29345494134,
        "mean_ms": 156.09305390425533,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          21,
          22,
          29,
          45,
          26,
          27,
          12,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 677.377175
      },
      "completed_queue_wait": {
        "count": 189,
        "sum_ns": 10417649076,
        "mean_ms": 55.11983638095238,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          20,
          18,
          31,
          17,
          12,
          12,
          25,
          14,
          24,
          7,
          5,
          4,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 476.733871
      },
      "preprocess_start_to_preprocess_output": {
        "count": 186,
        "sum_ns": 3743447158,
        "mean_ms": 20.12605998924731,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          2,
          7,
          16,
          37,
          68,
          44,
          8,
          1,
          2,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 369.690747
      },
      "selected_to_preprocess": {
        "count": 186,
        "sum_ns": 9052623201,
        "mean_ms": 48.67001720967742,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          2,
          7,
          23,
          100,
          31,
          13,
          7,
          0,
          0,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 602.991246
      },
      "preprocess_start_to_preprocess": {
        "count": 186,
        "sum_ns": 7009205263,
        "mean_ms": 37.683899263440864,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          7,
          12,
          45,
          89,
          20,
          5,
          3,
          0,
          0,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 588.13986
      },
      "preprocess_output_to_infer_entry": {
        "count": 186,
        "sum_ns": 3265758105,
        "mean_ms": 17.557839274193547,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          32,
          15,
          20,
          32,
          47,
          33,
          4,
          1,
          0,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 549.183919
      },
      "preprocess_to_infer_output": {
        "count": 184,
        "sum_ns": 94933831030,
        "mean_ms": 515.9447338586956,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          15,
          75,
          90,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1276.877611
      },
      "preprocess_to_callback": {
        "count": 185,
        "sum_ns": 97938515890,
        "mean_ms": 529.3973831891892,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          76,
          93,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1278.646734
      },
      "infer_output_to_callback": {
        "count": 184,
        "sum_ns": 2468185860,
        "mean_ms": 13.414053586956522,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          10,
          34,
          43,
          60,
          35,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 70.1736
      },
      "callback_to_copied": {
        "count": 185,
        "sum_ns": 10698559788,
        "mean_ms": 57.83005290810811,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          25,
          73,
          49,
          16,
          12,
          1,
          0,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 592.111576
      },
      "selected_to_submitted": {
        "count": 185,
        "sum_ns": 118619912247,
        "mean_ms": 641.1887148486487,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          48,
          129,
          7,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1416.975597
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.61393053177744,
    "first_redis_commit_s": 18.744325953070074,
    "maximum_sampled_silence_ms": 583.082005,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 288,
    "accepted": 288,
    "sent": 239,
    "late": 0,
    "completed_fps": 9.599256417683558,
    "accepted_fps": 9.599256417683558,
    "sent_fps": 7.9660495966193405,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.638847616,
    "selected": 289,
    "infer_entries": 288,
    "infer_entry_fps": 9.716976980053987,
    "pre_infer_expired": 0,
    "diagnostic_missed": 10,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 286,
        "sum_ns": 2489787771,
        "mean_ms": 8.705551646853147,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          66,
          88,
          45,
          56,
          29,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.033023
      },
      "preprocess_start_to_preprocess_output": {
        "count": 284,
        "sum_ns": 3818743836,
        "mean_ms": 13.446281112676056,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          9,
          19,
          85,
          128,
          37,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 82.584267
      },
      "selected_to_preprocess": {
        "count": 285,
        "sum_ns": 8965544304,
        "mean_ms": 31.458050189473685,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          5,
          15,
          64,
          166,
          25,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 125.434628
      },
      "preprocess_start_to_preprocess": {
        "count": 285,
        "sum_ns": 6479561799,
        "mean_ms": 22.735304557894736,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          8,
          38,
          98,
          125,
          9,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 91.167238
      },
      "preprocess_output_to_infer_entry": {
        "count": 284,
        "sum_ns": 2644939966,
        "mean_ms": 9.313168894366198,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          60,
          23,
          39,
          61,
          69,
          30,
          0,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.773416
      },
      "accepted_interval": {
        "count": 288,
        "sum_ns": 29619920304,
        "mean_ms": 102.8469455,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          28,
          74,
          74,
          70,
          20,
          18,
          4,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 334.02018
      },
      "completed_queue_wait": {
        "count": 288,
        "sum_ns": 1143111773,
        "mean_ms": 3.9691381006944444,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          77,
          56,
          76,
          56,
          22,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.419055
      },
      "copy_wall": {
        "count": 288,
        "sum_ns": 15237795680,
        "mean_ms": 52.909012777777775,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          151,
          76,
          39,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 199.366333
      },
      "copy_thread_cpu": {
        "count": 288,
        "sum_ns": 5809066169,
        "mean_ms": 20.17036864236111,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          142,
          145,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 153.964237
      },
      "preprocess_to_infer_output": {
        "count": 283,
        "sum_ns": 93940152404,
        "mean_ms": 331.94400142756183,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          15,
          92,
          159,
          16,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 575.173123
      },
      "preprocess_to_callback": {
        "count": 284,
        "sum_ns": 97317649318,
        "mean_ms": 342.6677792887324,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          7,
          88,
          168,
          20,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 595.959447
      },
      "infer_output_to_callback": {
        "count": 284,
        "sum_ns": 3093513991,
        "mean_ms": 10.892654897887324,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          55,
          97,
          106,
          24,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.18787
      },
      "callback_to_copied": {
        "count": 286,
        "sum_ns": 15584242907,
        "mean_ms": 54.49035981468531,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          8,
          142,
          77,
          42,
          15,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 200.265926
      },
      "selected_to_submitted": {
        "count": 286,
        "sum_ns": 122806248638,
        "mean_ms": 429.3924777552447,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          27,
          188,
          71,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 755.72195
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.428674277849495,
    "first_redis_commit_s": 18.527694292832166,
    "maximum_sampled_silence_ms": 81.894794,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 282,
    "accepted": 282,
    "sent": 209,
    "late": 0,
    "completed_fps": 9.399271908981817,
    "accepted_fps": 9.399271908981817,
    "sent_fps": 6.966127053110637,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.711812307,
    "selected": 282,
    "infer_entries": 281,
    "infer_entry_fps": 9.457518009892564,
    "pre_infer_expired": 0,
    "diagnostic_missed": 9,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 280,
        "sum_ns": 1887048085,
        "mean_ms": 6.739457446428572,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          79,
          80,
          51,
          57,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 46.561876
      },
      "copy_wall": {
        "count": 282,
        "sum_ns": 12733054798,
        "mean_ms": 45.15267658865248,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          17,
          171,
          66,
          24,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.248848
      },
      "copy_thread_cpu": {
        "count": 282,
        "sum_ns": 5299382508,
        "mean_ms": 18.79213655319149,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          170,
          112,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.62024
      },
      "accepted_interval": {
        "count": 282,
        "sum_ns": 29606098007,
        "mean_ms": 104.98616314539007,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          50,
          74,
          42,
          55,
          37,
          21,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 430.444279
      },
      "completed_queue_wait": {
        "count": 282,
        "sum_ns": 822416576,
        "mean_ms": 2.9163708368794325,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          86,
          63,
          91,
          32,
          7,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 36.55344
      },
      "preprocess_start_to_preprocess_output": {
        "count": 277,
        "sum_ns": 2864338834,
        "mean_ms": 10.34057340794224,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          14,
          34,
          102,
          106,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 36.995296
      },
      "selected_to_preprocess": {
        "count": 280,
        "sum_ns": 6842824625,
        "mean_ms": 24.438659375,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          3,
          23,
          94,
          147,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 80.914349
      },
      "preprocess_start_to_preprocess": {
        "count": 279,
        "sum_ns": 4940945342,
        "mean_ms": 17.709481512544805,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          20,
          56,
          104,
          94,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 79.242104
      },
      "preprocess_output_to_infer_entry": {
        "count": 277,
        "sum_ns": 2046097489,
        "mean_ms": 7.386633534296029,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          70,
          17,
          54,
          55,
          61,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.304994
      },
      "preprocess_to_infer_output": {
        "count": 280,
        "sum_ns": 97996070457,
        "mean_ms": 349.9859659178572,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          7,
          83,
          163,
          23,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 640.273015
      },
      "preprocess_to_callback": {
        "count": 280,
        "sum_ns": 100361493046,
        "mean_ms": 358.43390373571424,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          69,
          176,
          24,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 645.548666
      },
      "infer_output_to_callback": {
        "count": 282,
        "sum_ns": 2383841679,
        "mean_ms": 8.453339287234042,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          7,
          74,
          116,
          73,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 38.207599
      },
      "callback_to_copied": {
        "count": 281,
        "sum_ns": 13109997476,
        "mean_ms": 46.65479528825623,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          175,
          64,
          26,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 140.183128
      },
      "selected_to_submitted": {
        "count": 281,
        "sum_ns": 121087488915,
        "mean_ms": 430.916330658363,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          22,
          194,
          64,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 750.082081
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.481112030800432,
    "first_redis_commit_s": 20.354287057649344,
    "maximum_sampled_silence_ms": 168.732277,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 183,
    "accepted": 183,
    "sent": 154,
    "late": 0,
    "completed_fps": 6.099527515403094,
    "accepted_fps": 6.099527515403094,
    "sent_fps": 5.13293572334468,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.005835625,
    "selected": 183,
    "infer_entries": 182,
    "infer_entry_fps": 6.065486803119152,
    "pre_infer_expired": 0,
    "diagnostic_missed": 10,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 178,
        "sum_ns": 2314825994,
        "mean_ms": 13.004640415730337,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          36,
          49,
          21,
          28,
          38,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.056121
      },
      "preprocess_start_to_preprocess_output": {
        "count": 178,
        "sum_ns": 3819547928,
        "mean_ms": 21.45813442696629,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          14,
          45,
          68,
          37,
          10,
          1,
          2,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 524.165265
      },
      "copy_wall": {
        "count": 183,
        "sum_ns": 12329640147,
        "mean_ms": 67.37508277049182,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          13,
          70,
          48,
          24,
          21,
          3,
          0,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 609.996287
      },
      "copy_thread_cpu": {
        "count": 183,
        "sum_ns": 4510433103,
        "mean_ms": 24.647175426229506,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          7,
          102,
          68,
          4,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 421.58678
      },
      "accepted_interval": {
        "count": 183,
        "sum_ns": 30038505277,
        "mean_ms": 164.14483757923497,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          10,
          29,
          22,
          49,
          30,
          24,
          12,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 838.140387
      },
      "selected_to_preprocess": {
        "count": 179,
        "sum_ns": 9327942549,
        "mean_ms": 52.11141088826816,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          24,
          90,
          34,
          11,
          14,
          2,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 532.218676
      },
      "preprocess_start_to_preprocess": {
        "count": 177,
        "sum_ns": 6957159695,
        "mean_ms": 39.30598697740113,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          3,
          14,
          47,
          73,
          25,
          5,
          6,
          2,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 530.539344
      },
      "preprocess_output_to_infer_entry": {
        "count": 179,
        "sum_ns": 3161624405,
        "mean_ms": 17.662706173184358,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          22,
          8,
          27,
          35,
          42,
          32,
          9,
          1,
          1,
          1,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 354.345101
      },
      "completed_queue_wait": {
        "count": 183,
        "sum_ns": 12877685601,
        "mean_ms": 70.36986667213115,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          14,
          12,
          25,
          10,
          12,
          27,
          20,
          14,
          24,
          17,
          3,
          2,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 644.177774
      },
      "preprocess_to_infer_output": {
        "count": 179,
        "sum_ns": 95579133917,
        "mean_ms": 533.9616419944134,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          11,
          78,
          86,
          4,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1066.275161
      },
      "preprocess_to_callback": {
        "count": 178,
        "sum_ns": 97692867144,
        "mean_ms": 548.836332269663,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          68,
          96,
          5,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1090.839392
      },
      "infer_output_to_callback": {
        "count": 179,
        "sum_ns": 2748776633,
        "mean_ms": 15.356294039106146,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          22,
          41,
          73,
          37,
          1,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 121.653813
      },
      "callback_to_copied": {
        "count": 179,
        "sum_ns": 12442958651,
        "mean_ms": 69.51373548044694,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          9,
          68,
          46,
          28,
          20,
          4,
          0,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 611.329367
      },
      "selected_to_submitted": {
        "count": 180,
        "sum_ns": 121642186932,
        "mean_ms": 675.7899274,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          39,
          126,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1207.08201
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.413173242937773,
    "first_redis_commit_s": 19.5900968038477,
    "maximum_sampled_silence_ms": 226.165961,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 292,
    "accepted": 292,
    "sent": 239,
    "late": 0,
    "completed_fps": 9.732579423484717,
    "accepted_fps": 9.732579423484717,
    "sent_fps": 7.9660495966193405,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.721122389,
    "selected": 293,
    "infer_entries": 292,
    "infer_entry_fps": 9.824662614628284,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 293,
        "sum_ns": 2114692449,
        "mean_ms": 7.217380372013651,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          88,
          76,
          53,
          54,
          21,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.347148
      },
      "copy_wall": {
        "count": 292,
        "sum_ns": 16056203242,
        "mean_ms": 54.98699740410959,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          131,
          93,
          46,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 152.582587
      },
      "copy_thread_cpu": {
        "count": 292,
        "sum_ns": 5653206503,
        "mean_ms": 19.360296243150685,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          153,
          139,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.79424
      },
      "accepted_interval": {
        "count": 292,
        "sum_ns": 29728299137,
        "mean_ms": 101.80924361986303,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          31,
          70,
          74,
          70,
          35,
          9,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 376.689905
      },
      "preprocess_start_to_preprocess_output": {
        "count": 293,
        "sum_ns": 3959073918,
        "mean_ms": 13.512197672354949,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          2,
          5,
          30,
          89,
          112,
          52,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.490233
      },
      "completed_queue_wait": {
        "count": 291,
        "sum_ns": 1168873601,
        "mean_ms": 4.0167477697594505,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          66,
          70,
          82,
          48,
          19,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.301384
      },
      "selected_to_preprocess": {
        "count": 292,
        "sum_ns": 8742280749,
        "mean_ms": 29.939317633561643,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          4,
          14,
          70,
          171,
          26,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 103.809029
      },
      "preprocess_start_to_preprocess": {
        "count": 292,
        "sum_ns": 6629620946,
        "mean_ms": 22.704181321917808,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          4,
          14,
          33,
          97,
          129,
          9,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 92.084231
      },
      "preprocess_output_to_infer_entry": {
        "count": 292,
        "sum_ns": 2679133667,
        "mean_ms": 9.175115297945204,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          63,
          21,
          48,
          65,
          68,
          21,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 79.710768
      },
      "preprocess_to_infer_output": {
        "count": 291,
        "sum_ns": 95536382141,
        "mean_ms": 328.30371869759455,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          119,
          154,
          14,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 612.48393
      },
      "preprocess_to_callback": {
        "count": 292,
        "sum_ns": 99204177737,
        "mean_ms": 339.7403347157534,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          104,
          171,
          15,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 619.387045
      },
      "infer_output_to_callback": {
        "count": 292,
        "sum_ns": 3450966591,
        "mean_ms": 11.81837873630137,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          52,
          96,
          102,
          39,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.329408
      },
      "callback_to_copied": {
        "count": 292,
        "sum_ns": 16502446154,
        "mean_ms": 56.51522655479452,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          128,
          92,
          51,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 153.833034
      },
      "selected_to_submitted": {
        "count": 292,
        "sum_ns": 124646310268,
        "mean_ms": 426.8709255753425,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          224,
          55,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 684.899708
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.07460933784023,
    "first_redis_commit_s": 19.14658698020503,
    "maximum_sampled_silence_ms": 104.416244,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 276,
    "accepted": 276,
    "sent": 226,
    "late": 0,
    "completed_fps": 9.199287400280076,
    "accepted_fps": 9.199287400280076,
    "sent_fps": 7.532749827765569,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.696238434,
    "selected": 276,
    "infer_entries": 275,
    "infer_entry_fps": 9.260432112005986,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 275,
        "sum_ns": 2018393078,
        "mean_ms": 7.339611192727272,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          71,
          86,
          46,
          49,
          22,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 51.682789
      },
      "preprocess_start_to_preprocess_output": {
        "count": 275,
        "sum_ns": 3156028168,
        "mean_ms": 11.476466065454547,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          4,
          17,
          32,
          75,
          110,
          37,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.226085
      },
      "selected_to_preprocess": {
        "count": 274,
        "sum_ns": 7372523532,
        "mean_ms": 26.907020189781022,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          7,
          20,
          74,
          152,
          17,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 120.802702
      },
      "preprocess_start_to_preprocess": {
        "count": 274,
        "sum_ns": 5356645332,
        "mean_ms": 19.549800481751827,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          9,
          24,
          34,
          98,
          100,
          8,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 106.193212
      },
      "preprocess_output_to_infer_entry": {
        "count": 274,
        "sum_ns": 2208385856,
        "mean_ms": 8.059802394160583,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          83,
          24,
          27,
          66,
          52,
          17,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 79.00937
      },
      "copy_wall": {
        "count": 276,
        "sum_ns": 13716076244,
        "mean_ms": 49.69592842028985,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          149,
          77,
          35,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 122.300243
      },
      "copy_thread_cpu": {
        "count": 276,
        "sum_ns": 5544612216,
        "mean_ms": 20.089174695652176,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          125,
          151,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.05467
      },
      "accepted_interval": {
        "count": 276,
        "sum_ns": 29626176990,
        "mean_ms": 107.34122097826088,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          30,
          76,
          53,
          63,
          31,
          19,
          4,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 394.133673
      },
      "completed_queue_wait": {
        "count": 276,
        "sum_ns": 943324980,
        "mean_ms": 3.4178441304347826,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          65,
          71,
          79,
          46,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.665275
      },
      "preprocess_to_infer_output": {
        "count": 274,
        "sum_ns": 97334494669,
        "mean_ms": 355.23538200364965,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          7,
          67,
          177,
          22,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 674.040425
      },
      "preprocess_to_callback": {
        "count": 274,
        "sum_ns": 100517602542,
        "mean_ms": 366.8525640218978,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          7,
          57,
          184,
          26,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 691.091864
      },
      "infer_output_to_callback": {
        "count": 275,
        "sum_ns": 3193366495,
        "mean_ms": 11.612241800000001,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          55,
          86,
          92,
          39,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.919413
      },
      "callback_to_copied": {
        "count": 275,
        "sum_ns": 14136506377,
        "mean_ms": 51.40547773454546,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          145,
          79,
          37,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 125.791888
      },
      "selected_to_submitted": {
        "count": 275,
        "sum_ns": 122662374142,
        "mean_ms": 446.04499688,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          18,
          180,
          77,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 778.4587
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.02880530897528,
    "first_redis_commit_s": 23.1151128471829,
    "maximum_sampled_silence_ms": 120.46652,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 188,
    "accepted": 188,
    "sent": 140,
    "late": 0,
    "completed_fps": 6.266181272654544,
    "accepted_fps": 6.266181272654544,
    "sent_fps": 4.6663052030406185,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.003013365,
    "selected": 187,
    "infer_entries": 185,
    "infer_entry_fps": 6.166047314961092,
    "pre_infer_expired": 0,
    "diagnostic_missed": 9,
    "durations": {
      "copy_wall": {
        "count": 188,
        "sum_ns": 10835237139,
        "mean_ms": 57.63424010106383,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          3,
          29,
          96,
          31,
          17,
          8,
          0,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 924.855162
      },
      "copy_thread_cpu": {
        "count": 188,
        "sum_ns": 4719642276,
        "mean_ms": 25.104480191489362,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          25,
          121,
          37,
          1,
          0,
          0,
          0,
          0,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 536.725049
      },
      "accepted_interval": {
        "count": 188,
        "sum_ns": 29488907732,
        "mean_ms": 156.85589219148937,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          21,
          34,
          28,
          49,
          20,
          14,
          14,
          7,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1030.132006
      },
      "selected_to_preprocess_start": {
        "count": 185,
        "sum_ns": 2573606511,
        "mean_ms": 13.911386545945946,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          19,
          50,
          30,
          43,
          35,
          5,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 85.379795
      },
      "completed_queue_wait": {
        "count": 187,
        "sum_ns": 16792508142,
        "mean_ms": 89.79950878074867,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          13,
          15,
          22,
          10,
          14,
          28,
          7,
          12,
          29,
          13,
          11,
          10,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 711.271944
      },
      "preprocess_start_to_preprocess_output": {
        "count": 183,
        "sum_ns": 4155704264,
        "mean_ms": 22.708766469945356,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          8,
          10,
          36,
          69,
          47,
          10,
          0,
          2,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 532.226264
      },
      "selected_to_preprocess": {
        "count": 184,
        "sum_ns": 10421803763,
        "mean_ms": 56.64023784239131,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          2,
          29,
          90,
          34,
          14,
          10,
          1,
          0,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 904.048206
      },
      "preprocess_start_to_preprocess": {
        "count": 183,
        "sum_ns": 7854928439,
        "mean_ms": 42.92310622404372,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          4,
          11,
          54,
          79,
          18,
          8,
          4,
          0,
          0,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 899.464485
      },
      "preprocess_output_to_infer_entry": {
        "count": 182,
        "sum_ns": 3722864469,
        "mean_ms": 20.45529928021978,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          27,
          12,
          20,
          47,
          39,
          31,
          3,
          1,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 858.306738
      },
      "preprocess_to_infer_output": {
        "count": 183,
        "sum_ns": 98858631096,
        "mean_ms": 540.211098885246,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          91,
          74,
          6,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1242.41742
      },
      "preprocess_to_callback": {
        "count": 183,
        "sum_ns": 102703737037,
        "mean_ms": 561.2226067595628,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          89,
          79,
          9,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1280.862086
      },
      "infer_output_to_callback": {
        "count": 186,
        "sum_ns": 3903223591,
        "mean_ms": 20.985073069892472,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          16,
          41,
          67,
          55,
          3,
          0,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 648.858897
      },
      "callback_to_copied": {
        "count": 185,
        "sum_ns": 10975906376,
        "mean_ms": 59.329223654054054,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          25,
          95,
          35,
          17,
          7,
          1,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 925.794001
      },
      "selected_to_submitted": {
        "count": 185,
        "sum_ns": 126086416171,
        "mean_ms": 681.5481955189189,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          53,
          108,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1549.164365
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.98433996969834,
    "first_redis_commit_s": 24.26703273691237,
    "maximum_sampled_silence_ms": 607.008224,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 293,
    "accepted": 293,
    "sent": 229,
    "late": 0,
    "completed_fps": 9.765910174935009,
    "accepted_fps": 9.765910174935009,
    "sent_fps": 7.632742082116439,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.750310833,
    "selected": 295,
    "infer_entries": 289,
    "infer_entry_fps": 9.71418421885636,
    "pre_infer_expired": 0,
    "diagnostic_missed": 7,
    "durations": {
      "copy_wall": {
        "count": 293,
        "sum_ns": 13757168979,
        "mean_ms": 46.952795150170644,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          186,
          65,
          21,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 147.060912
      },
      "copy_thread_cpu": {
        "count": 293,
        "sum_ns": 5881422603,
        "mean_ms": 20.073114686006825,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          142,
          151,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.64119
      },
      "accepted_interval": {
        "count": 293,
        "sum_ns": 29767416703,
        "mean_ms": 101.59527884982934,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          45,
          90,
          47,
          60,
          32,
          12,
          7,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 438.243617
      },
      "completed_queue_wait": {
        "count": 293,
        "sum_ns": 986330360,
        "mean_ms": 3.3663152218430032,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          69,
          74,
          93,
          39,
          15,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.029063
      },
      "selected_to_preprocess_start": {
        "count": 295,
        "sum_ns": 2796957981,
        "mean_ms": 9.481213494915254,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          49,
          84,
          51,
          81,
          27,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.023732
      },
      "preprocess_start_to_preprocess_output": {
        "count": 292,
        "sum_ns": 3594433857,
        "mean_ms": 12.309704989726027,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          10,
          19,
          88,
          141,
          33,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.698853
      },
      "selected_to_preprocess": {
        "count": 289,
        "sum_ns": 9073582025,
        "mean_ms": 31.39647759515571,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          4,
          11,
          58,
          183,
          27,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.787623
      },
      "preprocess_start_to_preprocess": {
        "count": 289,
        "sum_ns": 6370150058,
        "mean_ms": 22.042041723183388,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          6,
          8,
          23,
          114,
          127,
          8,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.631096
      },
      "preprocess_output_to_infer_entry": {
        "count": 288,
        "sum_ns": 2795428102,
        "mean_ms": 9.706347576388888,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          55,
          16,
          36,
          87,
          70,
          19,
          3,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 105.134317
      },
      "preprocess_to_infer_output": {
        "count": 288,
        "sum_ns": 96508562815,
        "mean_ms": 335.0991764409722,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          96,
          151,
          25,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 673.889022
      },
      "preprocess_to_callback": {
        "count": 288,
        "sum_ns": 100195618032,
        "mean_ms": 347.9014515,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          82,
          165,
          29,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 682.822457
      },
      "infer_output_to_callback": {
        "count": 293,
        "sum_ns": 3759352067,
        "mean_ms": 12.830553129692833,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          55,
          87,
          96,
          53,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 58.23316
      },
      "callback_to_copied": {
        "count": 292,
        "sum_ns": 14330456665,
        "mean_ms": 49.0769063869863,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          178,
          68,
          26,
          13,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 147.823198
      },
      "selected_to_submitted": {
        "count": 292,
        "sum_ns": 125785776333,
        "mean_ms": 430.77320661986306,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          35,
          195,
          62,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 775.025117
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.639692968688905,
    "first_redis_commit_s": 23.87110866094008,
    "maximum_sampled_silence_ms": 185.110989,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.002323874738067,
    "completed": 287,
    "accepted": 287,
    "sent": 212,
    "late": 0,
    "completed_fps": 9.565925666233268,
    "accepted_fps": 9.565925666233268,
    "sent_fps": 7.066119307461507,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.794039833,
    "selected": 288,
    "infer_entries": 286,
    "infer_entry_fps": 9.599235337103405,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 287,
        "sum_ns": 2951390184,
        "mean_ms": 10.283589491289199,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          43,
          74,
          39,
          89,
          41,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.689834
      },
      "preprocess_start_to_preprocess_output": {
        "count": 287,
        "sum_ns": 3933886348,
        "mean_ms": 13.706921073170731,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          4,
          88,
          146,
          44,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.294083
      },
      "selected_to_preprocess": {
        "count": 286,
        "sum_ns": 9364571363,
        "mean_ms": 32.74325651398601,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          3,
          43,
          211,
          24,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 110.858877
      },
      "preprocess_start_to_preprocess": {
        "count": 285,
        "sum_ns": 6387724723,
        "mean_ms": 22.413069203508773,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          3,
          18,
          119,
          137,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 89.837655
      },
      "preprocess_output_to_infer_entry": {
        "count": 286,
        "sum_ns": 2486524153,
        "mean_ms": 8.694140395104894,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          42,
          20,
          30,
          104,
          72,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.614683
      },
      "copy_wall": {
        "count": 287,
        "sum_ns": 13355699725,
        "mean_ms": 46.53553911149826,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          172,
          71,
          25,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.14999
      },
      "copy_thread_cpu": {
        "count": 287,
        "sum_ns": 5879598586,
        "mean_ms": 20.486406222996514,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          121,
          166,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.334427
      },
      "accepted_interval": {
        "count": 287,
        "sum_ns": 29732785829,
        "mean_ms": 103.59855689547038,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          44,
          77,
          57,
          58,
          28,
          19,
          4,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 384.6114
      },
      "completed_queue_wait": {
        "count": 287,
        "sum_ns": 889501866,
        "mean_ms": 3.099309637630662,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          90,
          74,
          65,
          44,
          11,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.419436
      },
      "preprocess_to_infer_output": {
        "count": 281,
        "sum_ns": 102447483309,
        "mean_ms": 364.5817911352313,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          7,
          58,
          187,
          26,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 681.761537
      },
      "preprocess_to_callback": {
        "count": 282,
        "sum_ns": 106873433176,
        "mean_ms": 378.9838055886525,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          7,
          42,
          200,
          31,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 694.822753
      },
      "infer_output_to_callback": {
        "count": 283,
        "sum_ns": 4076081460,
        "mean_ms": 14.403114699646643,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          44,
          73,
          99,
          64,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.307802
      },
      "callback_to_copied": {
        "count": 283,
        "sum_ns": 13773068921,
        "mean_ms": 48.66808806007067,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          162,
          77,
          24,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.954203
      },
      "selected_to_submitted": {
        "count": 283,
        "sum_ns": 130661192418,
        "mean_ms": 461.70032656537103,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          15,
          180,
          86,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 892.066688
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.647129592951387,
    "first_redis_commit_s": 28.29809491802007,
    "maximum_sampled_silence_ms": 250.372213,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 185,
    "accepted": 185,
    "sent": 148,
    "late": 0,
    "completed_fps": 6.166189018303674,
    "accepted_fps": 6.166189018303674,
    "sent_fps": 4.932951214642939,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.515080065,
    "selected": 186,
    "infer_entries": 185,
    "infer_entry_fps": 6.2679823192951245,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "copy_wall": {
        "count": 185,
        "sum_ns": 11274054458,
        "mean_ms": 60.94083490810811,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          25,
          77,
          52,
          10,
          14,
          1,
          1,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 865.447286
      },
      "copy_thread_cpu": {
        "count": 185,
        "sum_ns": 4193851605,
        "mean_ms": 22.669468135135137,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          7,
          122,
          53,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 468.726659
      },
      "accepted_interval": {
        "count": 185,
        "sum_ns": 29576242374,
        "mean_ms": 159.8715804,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          13,
          31,
          29,
          37,
          30,
          28,
          9,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 993.578859
      },
      "completed_queue_wait": {
        "count": 185,
        "sum_ns": 17327334300,
        "mean_ms": 93.66126648648648,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          18,
          14,
          23,
          13,
          7,
          13,
          14,
          13,
          21,
          17,
          23,
          8,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 792.71537
      },
      "selected_to_preprocess_start": {
        "count": 184,
        "sum_ns": 2347217757,
        "mean_ms": 12.756618244565217,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          33,
          53,
          21,
          40,
          32,
          3,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 674.616393
      },
      "preprocess_start_to_preprocess_output": {
        "count": 182,
        "sum_ns": 4781938642,
        "mean_ms": 26.27438814285714,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          2,
          6,
          36,
          76,
          47,
          10,
          1,
          1,
          2,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 855.292926
      },
      "selected_to_preprocess": {
        "count": 185,
        "sum_ns": 10579898994,
        "mean_ms": 57.18864321081081,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          3,
          18,
          98,
          39,
          11,
          6,
          6,
          2,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 859.888323
      },
      "preprocess_start_to_preprocess": {
        "count": 183,
        "sum_ns": 8146558760,
        "mean_ms": 44.51671453551912,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          1,
          2,
          12,
          37,
          89,
          25,
          4,
          8,
          3,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 857.996741
      },
      "preprocess_output_to_infer_entry": {
        "count": 184,
        "sum_ns": 3348793970,
        "mean_ms": 18.19996722826087,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          28,
          12,
          17,
          39,
          48,
          29,
          1,
          7,
          2,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 512.23588
      },
      "preprocess_to_infer_output": {
        "count": 186,
        "sum_ns": 102784713652,
        "mean_ms": 552.605987376344,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          75,
          94,
          5,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1183.454124
      },
      "preprocess_to_callback": {
        "count": 186,
        "sum_ns": 106241026677,
        "mean_ms": 571.188315467742,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          69,
          101,
          7,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1193.551447
      },
      "infer_output_to_callback": {
        "count": 186,
        "sum_ns": 3456313025,
        "mean_ms": 18.582328091397848,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          19,
          45,
          72,
          43,
          3,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 685.690728
      },
      "callback_to_copied": {
        "count": 185,
        "sum_ns": 11521239645,
        "mean_ms": 62.27697105405405,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          20,
          80,
          51,
          13,
          14,
          1,
          1,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 866.316641
      },
      "selected_to_submitted": {
        "count": 185,
        "sum_ns": 128826171834,
        "mean_ms": 696.3576855891891,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          38,
          126,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1385.197383
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.37550678802654,
    "first_redis_commit_s": 21.44207639945671,
    "maximum_sampled_silence_ms": 254.74583,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 296,
    "accepted": 296,
    "sent": 226,
    "late": 0,
    "completed_fps": 9.865902429285878,
    "accepted_fps": 9.865902429285878,
    "sent_fps": 7.532749827765569,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.903065653,
    "selected": 297,
    "infer_entries": 295,
    "infer_entry_fps": 9.86520925390151,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29994960099,
        "mean_ms": 101.33432465878379,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          51,
          84,
          49,
          56,
          28,
          25,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 399.072632
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 923927719,
        "mean_ms": 3.121377429054054,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          108,
          60,
          78,
          34,
          12,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.474173
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 13171636528,
        "mean_ms": 44.49877205405405,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          18,
          189,
          62,
          22,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 121.335915
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5681099090,
        "mean_ms": 19.192902331081083,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          160,
          136,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.330409
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 2449122434,
        "mean_ms": 8.274062277027026,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          57,
          96,
          50,
          71,
          21,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.153386
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 3245993106,
        "mean_ms": 10.966192925675676,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          5,
          18,
          34,
          87,
          127,
          24,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 57.162236
      },
      "selected_to_preprocess": {
        "count": 294,
        "sum_ns": 8177194008,
        "mean_ms": 27.81358506122449,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          13,
          28,
          68,
          156,
          23,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 102.100773
      },
      "preprocess_start_to_preprocess": {
        "count": 294,
        "sum_ns": 5744054191,
        "mean_ms": 19.537599289115644,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          13,
          24,
          41,
          94,
          116,
          5,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 100.837593
      },
      "preprocess_output_to_infer_entry": {
        "count": 294,
        "sum_ns": 2520171013,
        "mean_ms": 8.57201024829932,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          74,
          20,
          36,
          69,
          70,
          23,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 89.809686
      },
      "preprocess_to_infer_output": {
        "count": 293,
        "sum_ns": 94647992667,
        "mean_ms": 323.03069169624575,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          33,
          97,
          147,
          16,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 598.985877
      },
      "preprocess_to_callback": {
        "count": 293,
        "sum_ns": 97668192897,
        "mean_ms": 333.3385423105802,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          22,
          98,
          153,
          20,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 655.563108
      },
      "infer_output_to_callback": {
        "count": 295,
        "sum_ns": 3051413374,
        "mean_ms": 10.34377414915254,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          10,
          69,
          93,
          94,
          28,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 56.577231
      },
      "callback_to_copied": {
        "count": 295,
        "sum_ns": 13625988839,
        "mean_ms": 46.18979267457627,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          187,
          62,
          24,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 124.867167
      },
      "selected_to_submitted": {
        "count": 295,
        "sum_ns": 120578834862,
        "mean_ms": 408.74181309152544,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          55,
          171,
          69,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 714.498556
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.20394528610632,
    "first_redis_commit_s": 21.238167728297412,
    "maximum_sampled_silence_ms": 106.813465,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.002323874738067,
    "completed": 296,
    "accepted": 296,
    "sent": 204,
    "late": 0,
    "completed_fps": 9.865902429285878,
    "accepted_fps": 9.865902429285878,
    "sent_fps": 6.799473295859186,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.774417182,
    "selected": 296,
    "infer_entries": 297,
    "infer_entry_fps": 9.975006334617698,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 2333272208,
        "mean_ms": 7.882676378378378,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          70,
          88,
          48,
          64,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.509754
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 3325484214,
        "mean_ms": 11.272827844067796,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          4,
          13,
          27,
          97,
          128,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 46.919543
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 8042841182,
        "mean_ms": 27.080273340067343,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          7,
          21,
          73,
          173,
          20,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 96.930189
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 5705922039,
        "mean_ms": 19.21185871717172,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          10,
          16,
          37,
          115,
          111,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 95.483049
      },
      "preprocess_output_to_infer_entry": {
        "count": 295,
        "sum_ns": 2336992604,
        "mean_ms": 7.922008827118645,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          83,
          10,
          24,
          102,
          57,
          18,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 83.658076
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 12269055966,
        "mean_ms": 41.44951339864865,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          20,
          199,
          54,
          14,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 159.864041
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5568421218,
        "mean_ms": 18.812233844594594,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          173,
          123,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.122987
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29786052927,
        "mean_ms": 100.6285571858108,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          74,
          75,
          39,
          47,
          34,
          20,
          7,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 385.791074
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 734971377,
        "mean_ms": 2.4830114087837836,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          121,
          64,
          73,
          29,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.911741
      },
      "preprocess_to_infer_output": {
        "count": 296,
        "sum_ns": 103001402090,
        "mean_ms": 347.97770976351353,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          11,
          87,
          171,
          25,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 644.682333
      },
      "preprocess_to_callback": {
        "count": 296,
        "sum_ns": 106155084744,
        "mean_ms": 358.63204305405407,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          79,
          180,
          26,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 655.622382
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 3153682654,
        "mean_ms": 10.65433329054054,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          8,
          61,
          96,
          100,
          31,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 43.570921
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 12776607250,
        "mean_ms": 43.16421368243243,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          13,
          203,
          52,
          18,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 160.797162
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 127281461529,
        "mean_ms": 430.00493759797297,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          28,
          203,
          64,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 726.091176
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.152840240392834,
    "first_redis_commit_s": 22.59020359022543,
    "maximum_sampled_silence_ms": 239.374098,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 130,
    "accepted": 130,
    "sent": 60,
    "late": 0,
    "completed_fps": 4.333319990036498,
    "accepted_fps": 4.333319990036498,
    "sent_fps": 1.9999938415553065,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.208235843,
    "selected": 130,
    "infer_entries": 130,
    "infer_entry_fps": 4.303462164280084,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "copy_wall": {
        "count": 130,
        "sum_ns": 6692641356,
        "mean_ms": 51.481856584615386,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          21,
          57,
          24,
          15,
          11,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 429.888468
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2282036659,
        "mean_ms": 17.554128146153843,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          13,
          101,
          15,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 329.804307
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 30288454294,
        "mean_ms": 232.98810995384616,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          3,
          10,
          16,
          33,
          38,
          24,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 714.656243
      },
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 1615514098,
        "mean_ms": 12.427031523076923,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          33,
          26,
          31,
          13,
          9,
          8,
          4,
          1,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 153.283952
      },
      "selected_to_preprocess_start": {
        "count": 130,
        "sum_ns": 1397911282,
        "mean_ms": 10.753163707692309,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          31,
          38,
          23,
          18,
          14,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 79.243418
      },
      "preprocess_start_to_preprocess_output": {
        "count": 130,
        "sum_ns": 2730007576,
        "mean_ms": 21.000058276923074,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          3,
          7,
          26,
          55,
          33,
          3,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 480.919093
      },
      "selected_to_preprocess": {
        "count": 130,
        "sum_ns": 7976994977,
        "mean_ms": 61.361499823076926,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          2,
          21,
          59,
          20,
          9,
          11,
          2,
          3,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 590.813054
      },
      "preprocess_start_to_preprocess": {
        "count": 130,
        "sum_ns": 6579083695,
        "mean_ms": 50.60833611538462,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          2,
          10,
          33,
          52,
          14,
          6,
          5,
          2,
          3,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 589.033442
      },
      "preprocess_output_to_infer_entry": {
        "count": 130,
        "sum_ns": 3849076119,
        "mean_ms": 29.608277838461536,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          24,
          8,
          11,
          26,
          27,
          17,
          4,
          2,
          4,
          4,
          1,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 556.332245
      },
      "preprocess_to_infer_output": {
        "count": 129,
        "sum_ns": 105576227495,
        "mean_ms": 818.4203681782946,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          108,
          19,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1321.594465
      },
      "preprocess_to_callback": {
        "count": 130,
        "sum_ns": 107672336022,
        "mean_ms": 828.2487386307693,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          109,
          20,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1338.789976
      },
      "infer_output_to_callback": {
        "count": 130,
        "sum_ns": 1446859002,
        "mean_ms": 11.12968463076923,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          26,
          50,
          39,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 46.612971
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 6945844678,
        "mean_ms": 53.42957444615385,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          18,
          55,
          28,
          16,
          11,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 430.602074
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 123121408549,
        "mean_ms": 947.0877580692307,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          89,
          41,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1571.728813
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.61393053177744,
    "first_redis_commit_s": 18.744325953070074,
    "maximum_sampled_silence_ms": 215.443564,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 158,
    "accepted": 158,
    "sent": 59,
    "late": 0,
    "completed_fps": 5.266650449428973,
    "accepted_fps": 5.266650449428973,
    "sent_fps": 1.966660610862718,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.139478342,
    "selected": 157,
    "infer_entries": 157,
    "infer_entry_fps": 5.209114710562763,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 156,
        "sum_ns": 1513272888,
        "mean_ms": 9.70046723076923,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          35,
          43,
          19,
          38,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.053746
      },
      "preprocess_start_to_preprocess_output": {
        "count": 156,
        "sum_ns": 2159957628,
        "mean_ms": 13.84588223076923,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          7,
          48,
          78,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.744295
      },
      "selected_to_preprocess": {
        "count": 156,
        "sum_ns": 7450190394,
        "mean_ms": 47.75763073076923,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          14,
          96,
          22,
          5,
          7,
          5,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 223.653626
      },
      "preprocess_start_to_preprocess": {
        "count": 156,
        "sum_ns": 5900941129,
        "mean_ms": 37.82654569871795,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          2,
          13,
          40,
          72,
          13,
          3,
          7,
          4,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 222.489371
      },
      "preprocess_output_to_infer_entry": {
        "count": 155,
        "sum_ns": 3607242331,
        "mean_ms": 23.272531167741935,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          26,
          8,
          11,
          36,
          30,
          26,
          5,
          4,
          4,
          3,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 218.491859
      },
      "copy_wall": {
        "count": 158,
        "sum_ns": 10047663478,
        "mean_ms": 63.59280682278481,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          59,
          47,
          30,
          15,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 232.407946
      },
      "copy_thread_cpu": {
        "count": 158,
        "sum_ns": 3206295044,
        "mean_ms": 20.293006607594936,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          65,
          93,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.870404
      },
      "accepted_interval": {
        "count": 158,
        "sum_ns": 30198536810,
        "mean_ms": 191.12997981012657,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          9,
          18,
          30,
          44,
          35,
          19,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 526.804149
      },
      "completed_queue_wait": {
        "count": 158,
        "sum_ns": 460384399,
        "mean_ms": 2.9138253101265823,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          54,
          39,
          39,
          16,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.764291
      },
      "preprocess_to_infer_output": {
        "count": 157,
        "sum_ns": 107132282628,
        "mean_ms": 682.3712269299363,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          11,
          142,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1129.031831
      },
      "preprocess_to_callback": {
        "count": 157,
        "sum_ns": 108635061608,
        "mean_ms": 691.943067566879,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          143,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1136.190366
      },
      "infer_output_to_callback": {
        "count": 157,
        "sum_ns": 1502778980,
        "mean_ms": 9.571840636942675,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          29,
          68,
          46,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.483716
      },
      "callback_to_copied": {
        "count": 158,
        "sum_ns": 10450663378,
        "mean_ms": 66.14343910126583,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          53,
          51,
          30,
          17,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 247.688594
      },
      "selected_to_submitted": {
        "count": 158,
        "sum_ns": 127551215258,
        "mean_ms": 807.2861725189873,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          139,
          16,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1210.1496
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.428674277849495,
    "first_redis_commit_s": 18.527694292832166,
    "maximum_sampled_silence_ms": 192.02193,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 165,
    "accepted": 165,
    "sent": 58,
    "late": 0,
    "completed_fps": 5.499983064277092,
    "accepted_fps": 5.499983064277092,
    "sent_fps": 1.9333273801701294,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.078114708,
    "selected": 165,
    "infer_entries": 165,
    "infer_entry_fps": 5.48571616279242,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 164,
        "sum_ns": 1450777126,
        "mean_ms": 8.846201987804879,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          29,
          47,
          32,
          35,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.015547
      },
      "preprocess_start_to_preprocess_output": {
        "count": 164,
        "sum_ns": 2205757491,
        "mean_ms": 13.449740798780487,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          10,
          38,
          82,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.986048
      },
      "accepted_interval": {
        "count": 165,
        "sum_ns": 30029818392,
        "mean_ms": 181.99889934545453,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          8,
          15,
          42,
          37,
          45,
          16,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 446.740957
      },
      "completed_queue_wait": {
        "count": 165,
        "sum_ns": 358748313,
        "mean_ms": 2.1742322,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          69,
          34,
          45,
          14,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 21.707029
      },
      "selected_to_preprocess": {
        "count": 165,
        "sum_ns": 7013266305,
        "mean_ms": 42.504644272727276,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          2,
          32,
          90,
          20,
          8,
          8,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.449193
      },
      "preprocess_start_to_preprocess": {
        "count": 164,
        "sum_ns": 5543289418,
        "mean_ms": 33.80054523170732,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          2,
          4,
          10,
          50,
          73,
          8,
          7,
          6,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 215.977222
      },
      "preprocess_output_to_infer_entry": {
        "count": 164,
        "sum_ns": 3337531927,
        "mean_ms": 20.35080443292683,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          34,
          5,
          15,
          41,
          36,
          14,
          6,
          5,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 214.090342
      },
      "copy_wall": {
        "count": 165,
        "sum_ns": 9396582882,
        "mean_ms": 56.94898716363637,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          70,
          51,
          25,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 166.408017
      },
      "copy_thread_cpu": {
        "count": 165,
        "sum_ns": 3274588935,
        "mean_ms": 19.845993545454547,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          78,
          87,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.259026
      },
      "preprocess_to_infer_output": {
        "count": 166,
        "sum_ns": 107998198672,
        "mean_ms": 650.5915582650601,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          22,
          142,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1124.913402
      },
      "preprocess_to_callback": {
        "count": 165,
        "sum_ns": 109168433666,
        "mean_ms": 661.6268707030304,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          20,
          143,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1141.208844
      },
      "infer_output_to_callback": {
        "count": 165,
        "sum_ns": 1914873971,
        "mean_ms": 11.605296793939393,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          29,
          60,
          56,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 51.760298
      },
      "callback_to_copied": {
        "count": 165,
        "sum_ns": 9856252745,
        "mean_ms": 59.734865121212124,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          65,
          55,
          28,
          12,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.508704
      },
      "selected_to_submitted": {
        "count": 165,
        "sum_ns": 126292073829,
        "mean_ms": 765.4065080545454,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          7,
          145,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1255.510068
      }
    },
    "rejection_reasons": {},
    "first_response_s": 18.481112030800432,
    "first_redis_commit_s": 20.354287057649344,
    "maximum_sampled_silence_ms": 112.349395,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 123,
    "accepted": 123,
    "sent": 55,
    "late": 0,
    "completed_fps": 4.099987375188378,
    "accepted_fps": 4.099987375188378,
    "sent_fps": 1.8333276880923641,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.250297091,
    "selected": 123,
    "infer_entries": 123,
    "infer_entry_fps": 4.066075768776323,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "accepted_interval": {
        "count": 123,
        "sum_ns": 30079608042,
        "mean_ms": 244.54965887804877,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          2,
          8,
          20,
          26,
          31,
          29,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 825.627897
      },
      "completed_queue_wait": {
        "count": 123,
        "sum_ns": 2101087614,
        "mean_ms": 17.082013121951217,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          30,
          28,
          24,
          16,
          7,
          6,
          6,
          4,
          0,
          1,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 676.405109
      },
      "selected_to_preprocess_start": {
        "count": 123,
        "sum_ns": 1692865532,
        "mean_ms": 13.763134406504065,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          14,
          37,
          11,
          35,
          20,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.278135
      },
      "preprocess_start_to_preprocess_output": {
        "count": 123,
        "sum_ns": 2112506577,
        "mean_ms": 17.174850219512194,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          10,
          26,
          49,
          31,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 95.018847
      },
      "copy_wall": {
        "count": 123,
        "sum_ns": 7516233494,
        "mean_ms": 61.10758938211382,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          11,
          53,
          31,
          16,
          9,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 700.533931
      },
      "copy_thread_cpu": {
        "count": 123,
        "sum_ns": 2613946993,
        "mean_ms": 21.25160156910569,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          6,
          84,
          32,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 441.283313
      },
      "selected_to_preprocess": {
        "count": 123,
        "sum_ns": 7468990815,
        "mean_ms": 60.72350256097561,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          1,
          3,
          16,
          55,
          18,
          12,
          11,
          2,
          3,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 399.397316
      },
      "preprocess_start_to_preprocess": {
        "count": 123,
        "sum_ns": 5776125283,
        "mean_ms": 46.960368154471546,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          1,
          9,
          7,
          25,
          52,
          6,
          9,
          7,
          2,
          3,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 387.622067
      },
      "preprocess_output_to_infer_entry": {
        "count": 123,
        "sum_ns": 3663618706,
        "mean_ms": 29.785517934959348,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          32,
          4,
          13,
          23,
          19,
          10,
          6,
          8,
          2,
          1,
          3,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 364.791624
      },
      "preprocess_to_infer_output": {
        "count": 124,
        "sum_ns": 105741948221,
        "mean_ms": 852.7576469435485,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          100,
          22,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1487.570882
      },
      "preprocess_to_callback": {
        "count": 124,
        "sum_ns": 107429297767,
        "mean_ms": 866.3653045725807,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          98,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1501.067784
      },
      "infer_output_to_callback": {
        "count": 124,
        "sum_ns": 1687349546,
        "mean_ms": 13.607657629032257,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          22,
          40,
          40,
          17,
          3,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 140.577046
      },
      "callback_to_copied": {
        "count": 123,
        "sum_ns": 7829543409,
        "mean_ms": 63.654824463414634,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          10,
          47,
          35,
          19,
          9,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 701.725704
      },
      "selected_to_submitted": {
        "count": 123,
        "sum_ns": 122500195154,
        "mean_ms": 995.9365459674797,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          69,
          54,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1675.541126
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.413173242937773,
    "first_redis_commit_s": 19.5900968038477,
    "maximum_sampled_silence_ms": 328.39219,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 158,
    "accepted": 158,
    "sent": 59,
    "late": 0,
    "completed_fps": 5.266650449428973,
    "accepted_fps": 5.266650449428973,
    "sent_fps": 1.966660610862718,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.066492993,
    "selected": 157,
    "infer_entries": 157,
    "infer_entry_fps": 5.22175965239951,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 158,
        "sum_ns": 10733436385,
        "mean_ms": 67.93314167721519,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          57,
          46,
          22,
          28,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.26341
      },
      "copy_thread_cpu": {
        "count": 158,
        "sum_ns": 3262424223,
        "mean_ms": 20.648254575949366,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          60,
          98,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.799156
      },
      "accepted_interval": {
        "count": 158,
        "sum_ns": 30129999207,
        "mean_ms": 190.69619751265824,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          4,
          14,
          32,
          43,
          44,
          18,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 487.662159
      },
      "completed_queue_wait": {
        "count": 158,
        "sum_ns": 466706561,
        "mean_ms": 2.9538389936708858,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          64,
          27,
          37,
          20,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.748162
      },
      "selected_to_preprocess_start": {
        "count": 157,
        "sum_ns": 1570573356,
        "mean_ms": 10.003651949044585,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          31,
          41,
          25,
          36,
          23,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.374068
      },
      "preprocess_start_to_preprocess_output": {
        "count": 158,
        "sum_ns": 2419276950,
        "mean_ms": 15.311879430379747,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          7,
          44,
          74,
          30,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 62.276792
      },
      "selected_to_preprocess": {
        "count": 157,
        "sum_ns": 6905531570,
        "mean_ms": 43.984277515923566,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          1,
          32,
          78,
          28,
          9,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 223.067748
      },
      "preprocess_start_to_preprocess": {
        "count": 157,
        "sum_ns": 5374343055,
        "mean_ms": 34.23148442675159,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          3,
          8,
          50,
          69,
          17,
          5,
          4,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.041069
      },
      "preprocess_output_to_infer_entry": {
        "count": 157,
        "sum_ns": 2968172726,
        "mean_ms": 18.90555876433121,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          25,
          11,
          16,
          29,
          32,
          28,
          9,
          2,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 188.879623
      },
      "preprocess_to_infer_output": {
        "count": 156,
        "sum_ns": 105481372195,
        "mean_ms": 676.1626422756411,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          140,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1062.784341
      },
      "preprocess_to_callback": {
        "count": 156,
        "sum_ns": 107294863377,
        "mean_ms": 687.78758575,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          141,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1085.331451
      },
      "infer_output_to_callback": {
        "count": 157,
        "sum_ns": 1825930564,
        "mean_ms": 11.63013098089172,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          35,
          53,
          46,
          22,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.176685
      },
      "callback_to_copied": {
        "count": 158,
        "sum_ns": 11198648940,
        "mean_ms": 70.87752493670887,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          52,
          48,
          26,
          24,
          7,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 172.816029
      },
      "selected_to_submitted": {
        "count": 158,
        "sum_ns": 127250718097,
        "mean_ms": 805.3842917531646,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          145,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1273.273846
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.07460933784023,
    "first_redis_commit_s": 19.14658698020503,
    "maximum_sampled_silence_ms": 253.158154,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 162,
    "accepted": 162,
    "sent": 57,
    "late": 0,
    "completed_fps": 5.399983372199327,
    "accepted_fps": 5.399983372199327,
    "sent_fps": 1.899994149477541,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.888247527,
    "selected": 162,
    "infer_entries": 162,
    "infer_entry_fps": 5.420190657001714,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 162,
        "sum_ns": 1589429529,
        "mean_ms": 9.811293388888888,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          39,
          47,
          19,
          32,
          22,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 62.421533
      },
      "copy_wall": {
        "count": 162,
        "sum_ns": 9876433652,
        "mean_ms": 60.965639827160494,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          67,
          51,
          18,
          17,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 245.606882
      },
      "copy_thread_cpu": {
        "count": 162,
        "sum_ns": 3324665856,
        "mean_ms": 20.522628740740743,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          63,
          99,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.196193
      },
      "accepted_interval": {
        "count": 162,
        "sum_ns": 29861632896,
        "mean_ms": 184.33106725925924,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          10,
          21,
          23,
          39,
          48,
          13,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 697.094258
      },
      "preprocess_start_to_preprocess_output": {
        "count": 162,
        "sum_ns": 2145098611,
        "mean_ms": 13.241349450617284,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          13,
          53,
          67,
          27,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.863632
      },
      "selected_to_preprocess": {
        "count": 162,
        "sum_ns": 6859983790,
        "mean_ms": 42.34557895061728,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          5,
          26,
          85,
          26,
          11,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 211.974443
      },
      "preprocess_start_to_preprocess": {
        "count": 162,
        "sum_ns": 5270554261,
        "mean_ms": 32.534285561728396,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          5,
          11,
          52,
          66,
          16,
          5,
          4,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 209.877054
      },
      "preprocess_output_to_infer_entry": {
        "count": 162,
        "sum_ns": 3125455650,
        "mean_ms": 19.29293611111111,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          33,
          4,
          12,
          37,
          35,
          27,
          4,
          4,
          5,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 205.904072
      },
      "completed_queue_wait": {
        "count": 162,
        "sum_ns": 464404346,
        "mean_ms": 2.8666934938271607,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          70,
          29,
          27,
          28,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.129625
      },
      "preprocess_to_infer_output": {
        "count": 162,
        "sum_ns": 106259324125,
        "mean_ms": 655.9217538580248,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          28,
          131,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1085.12033
      },
      "preprocess_to_callback": {
        "count": 162,
        "sum_ns": 108146274696,
        "mean_ms": 667.5695968888888,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          23,
          135,
          4,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1087.635237
      },
      "infer_output_to_callback": {
        "count": 162,
        "sum_ns": 1886950571,
        "mean_ms": 11.647843030864198,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          29,
          58,
          52,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.83037
      },
      "callback_to_copied": {
        "count": 162,
        "sum_ns": 10287779299,
        "mean_ms": 63.50481048765432,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          61,
          54,
          18,
          21,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 246.201678
      },
      "selected_to_submitted": {
        "count": 162,
        "sum_ns": 125522181568,
        "mean_ms": 774.8282812839506,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          146,
          11,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1245.879782
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.02880530897528,
    "first_redis_commit_s": 23.1151128471829,
    "maximum_sampled_silence_ms": 210.498874,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 121,
    "accepted": 121,
    "sent": 47,
    "late": 0,
    "completed_fps": 4.033320913803201,
    "accepted_fps": 4.033320913803201,
    "sent_fps": 1.5666618425516567,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.300265194,
    "selected": 121,
    "infer_entries": 120,
    "infer_entry_fps": 3.960361377423263,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 121,
        "sum_ns": 2427663488,
        "mean_ms": 20.063334611570248,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          7,
          32,
          20,
          10,
          40,
          7,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.524532
      },
      "preprocess_start_to_preprocess_output": {
        "count": 120,
        "sum_ns": 3045752591,
        "mean_ms": 25.381271591666664,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          2,
          6,
          25,
          43,
          36,
          6,
          0,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 684.490506
      },
      "copy_wall": {
        "count": 121,
        "sum_ns": 8334276082,
        "mean_ms": 68.87831472727274,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          51,
          35,
          16,
          10,
          2,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 737.246314
      },
      "copy_thread_cpu": {
        "count": 121,
        "sum_ns": 2850157520,
        "mean_ms": 23.55502082644628,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          1,
          68,
          51,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 467.575183
      },
      "accepted_interval": {
        "count": 121,
        "sum_ns": 30192995136,
        "mean_ms": 249.5288854214876,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          3,
          7,
          26,
          23,
          35,
          17,
          8,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1014.891959
      },
      "selected_to_preprocess": {
        "count": 120,
        "sum_ns": 9271394041,
        "mean_ms": 77.26161700833335,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          1,
          1,
          9,
          48,
          30,
          11,
          11,
          3,
          2,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 895.548413
      },
      "preprocess_start_to_preprocess": {
        "count": 120,
        "sum_ns": 6845635769,
        "mean_ms": 57.04696474166667,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          1,
          1,
          7,
          28,
          56,
          8,
          8,
          6,
          0,
          1,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 893.028459
      },
      "preprocess_output_to_infer_entry": {
        "count": 120,
        "sum_ns": 3799883178,
        "mean_ms": 31.66569315,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          14,
          5,
          11,
          29,
          29,
          18,
          5,
          1,
          3,
          0,
          2,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 761.79709
      },
      "completed_queue_wait": {
        "count": 121,
        "sum_ns": 4843228961,
        "mean_ms": 40.02668562809917,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          28,
          22,
          27,
          8,
          10,
          7,
          2,
          1,
          7,
          4,
          3,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 970.236591
      },
      "preprocess_to_infer_output": {
        "count": 120,
        "sum_ns": 106237831298,
        "mean_ms": 885.3152608166668,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          88,
          30,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1696.078882
      },
      "preprocess_to_callback": {
        "count": 120,
        "sum_ns": 108494723562,
        "mean_ms": 904.1226963500001,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          89,
          30,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1740.403571
      },
      "infer_output_to_callback": {
        "count": 121,
        "sum_ns": 2317200858,
        "mean_ms": 19.150420314049587,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          19,
          34,
          28,
          32,
          4,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 147.991105
      },
      "callback_to_copied": {
        "count": 121,
        "sum_ns": 8716132232,
        "mean_ms": 72.03415067768594,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          48,
          33,
          20,
          11,
          3,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 738.087251
      },
      "selected_to_submitted": {
        "count": 121,
        "sum_ns": 127901076699,
        "mean_ms": 1057.0336917272728,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          60,
          60,
          1,
          0
        ],
        "diagnostic_window_max_ms": 2077.178404
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.98433996969834,
    "first_redis_commit_s": 24.26703273691237,
    "maximum_sampled_silence_ms": 341.503764,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 159,
    "accepted": 159,
    "sent": 61,
    "late": 0,
    "completed_fps": 5.299983680121562,
    "accepted_fps": 5.299983680121562,
    "sent_fps": 2.0333270722478947,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.116490405,
    "selected": 160,
    "infer_entries": 158,
    "infer_entry_fps": 5.24629523145793,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 159,
        "sum_ns": 2278648360,
        "mean_ms": 14.33112176100629,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          30,
          32,
          20,
          34,
          35,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.844144
      },
      "preprocess_start_to_preprocess_output": {
        "count": 159,
        "sum_ns": 2350436575,
        "mean_ms": 14.78261996855346,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          2,
          33,
          92,
          30,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.694397
      },
      "selected_to_preprocess": {
        "count": 157,
        "sum_ns": 7963759343,
        "mean_ms": 50.72458180254777,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          13,
          80,
          41,
          9,
          11,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 202.359074
      },
      "preprocess_start_to_preprocess": {
        "count": 157,
        "sum_ns": 5715553755,
        "mean_ms": 36.40480098726115,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          3,
          46,
          76,
          19,
          5,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 189.447539
      },
      "preprocess_output_to_infer_entry": {
        "count": 157,
        "sum_ns": 3383570745,
        "mean_ms": 21.55140601910828,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          20,
          2,
          13,
          35,
          39,
          31,
          9,
          4,
          2,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 177.316865
      },
      "copy_wall": {
        "count": 159,
        "sum_ns": 10070411400,
        "mean_ms": 63.33592075471698,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          68,
          44,
          27,
          14,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.162966
      },
      "copy_thread_cpu": {
        "count": 159,
        "sum_ns": 3250098480,
        "mean_ms": 20.440870943396227,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          70,
          89,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.247622
      },
      "accepted_interval": {
        "count": 159,
        "sum_ns": 30118171506,
        "mean_ms": 189.4224623018868,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          7,
          12,
          37,
          39,
          51,
          12,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 394.563504
      },
      "completed_queue_wait": {
        "count": 159,
        "sum_ns": 462962268,
        "mean_ms": 2.9117123773584908,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          58,
          28,
          41,
          26,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.177363
      },
      "preprocess_to_infer_output": {
        "count": 158,
        "sum_ns": 108500479533,
        "mean_ms": 686.711895778481,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          144,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 962.398517
      },
      "preprocess_to_callback": {
        "count": 158,
        "sum_ns": 110710265882,
        "mean_ms": 700.697885329114,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          146,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 984.048271
      },
      "infer_output_to_callback": {
        "count": 159,
        "sum_ns": 2220095414,
        "mean_ms": 13.96286423899371,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          19,
          48,
          64,
          26,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 75.012985
      },
      "callback_to_copied": {
        "count": 158,
        "sum_ns": 10606143301,
        "mean_ms": 67.12748924683544,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          61,
          43,
          30,
          17,
          5,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 228.53699
      },
      "selected_to_submitted": {
        "count": 158,
        "sum_ns": 129597038353,
        "mean_ms": 820.2344199556962,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          134,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1126.715909
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.639692968688905,
    "first_redis_commit_s": 23.87110866094008,
    "maximum_sampled_silence_ms": 152.503503,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 166,
    "accepted": 166,
    "sent": 58,
    "late": 0,
    "completed_fps": 5.533316294969681,
    "accepted_fps": 5.533316294969681,
    "sent_fps": 1.9333273801701294,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.283822944,
    "selected": 166,
    "infer_entries": 165,
    "infer_entry_fps": 5.448453463260348,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "accepted_interval": {
        "count": 166,
        "sum_ns": 30294963185,
        "mean_ms": 182.49977822289156,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          10,
          17,
          32,
          37,
          50,
          17,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 441.729845
      },
      "completed_queue_wait": {
        "count": 166,
        "sum_ns": 384496832,
        "mean_ms": 2.3162459759036143,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          63,
          43,
          43,
          14,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.511285
      },
      "selected_to_preprocess_start": {
        "count": 165,
        "sum_ns": 1859679985,
        "mean_ms": 11.270787787878787,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          25,
          46,
          23,
          43,
          27,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.871881
      },
      "preprocess_start_to_preprocess_output": {
        "count": 163,
        "sum_ns": 2463493860,
        "mean_ms": 15.113459263803682,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          2,
          39,
          82,
          37,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.368075
      },
      "selected_to_preprocess": {
        "count": 164,
        "sum_ns": 7006230657,
        "mean_ms": 42.7209186402439,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          12,
          105,
          31,
          11,
          0,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 194.728955
      },
      "preprocess_start_to_preprocess": {
        "count": 164,
        "sum_ns": 5148403046,
        "mean_ms": 31.3927015,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          8,
          43,
          90,
          14,
          5,
          0,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 187.253682
      },
      "preprocess_output_to_infer_entry": {
        "count": 163,
        "sum_ns": 2671263697,
        "mean_ms": 16.388120840490796,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          21,
          4,
          15,
          42,
          51,
          21,
          6,
          0,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 179.440528
      },
      "copy_wall": {
        "count": 166,
        "sum_ns": 9713860042,
        "mean_ms": 58.5172291686747,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          77,
          49,
          22,
          11,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 202.505689
      },
      "copy_thread_cpu": {
        "count": 166,
        "sum_ns": 3515702547,
        "mean_ms": 21.178931006024097,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          64,
          102,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.251535
      },
      "preprocess_to_infer_output": {
        "count": 164,
        "sum_ns": 108689820550,
        "mean_ms": 662.7428082317074,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          145,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 990.419687
      },
      "preprocess_to_callback": {
        "count": 164,
        "sum_ns": 110893981164,
        "mean_ms": 676.1828119756098,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          148,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 993.858984
      },
      "infer_output_to_callback": {
        "count": 165,
        "sum_ns": 2254062028,
        "mean_ms": 13.660981987878788,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          24,
          47,
          60,
          33,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 49.901414
      },
      "callback_to_copied": {
        "count": 165,
        "sum_ns": 10242462934,
        "mean_ms": 62.07553293333333,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          70,
          48,
          27,
          13,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 203.6496
      },
      "selected_to_submitted": {
        "count": 165,
        "sum_ns": 129285718792,
        "mean_ms": 783.549810860606,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          149,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1151.862897
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.647129592951387,
    "first_redis_commit_s": 28.29809491802007,
    "maximum_sampled_silence_ms": 166.066398,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 123,
    "accepted": 123,
    "sent": 51,
    "late": 0,
    "completed_fps": 4.099987375188378,
    "accepted_fps": 4.099987375188378,
    "sent_fps": 1.6999947653220104,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.886265942,
    "selected": 123,
    "infer_entries": 122,
    "infer_entry_fps": 4.082142621522684,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 123,
        "sum_ns": 1884494544,
        "mean_ms": 15.321093853658537,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          14,
          36,
          16,
          34,
          15,
          6,
          0,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 165.972612
      },
      "preprocess_start_to_preprocess_output": {
        "count": 122,
        "sum_ns": 2805766959,
        "mean_ms": 22.998089827868853,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          3,
          23,
          51,
          37,
          4,
          1,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 185.001347
      },
      "selected_to_preprocess": {
        "count": 122,
        "sum_ns": 9546790338,
        "mean_ms": 78.25237981967214,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          1,
          7,
          54,
          16,
          17,
          12,
          6,
          7,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 430.011217
      },
      "preprocess_start_to_preprocess": {
        "count": 122,
        "sum_ns": 7692742196,
        "mean_ms": 63.05526390163934,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          5,
          23,
          51,
          13,
          7,
          11,
          3,
          8,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 412.963118
      },
      "preprocess_output_to_infer_entry": {
        "count": 121,
        "sum_ns": 4884007664,
        "mean_ms": 40.36369970247934,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          14,
          5,
          10,
          21,
          32,
          13,
          4,
          7,
          6,
          2,
          6,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 407.322883
      },
      "copy_wall": {
        "count": 123,
        "sum_ns": 7360825677,
        "mean_ms": 59.844111195121954,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          65,
          23,
          16,
          8,
          2,
          1,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 648.46967
      },
      "copy_thread_cpu": {
        "count": 123,
        "sum_ns": 2758988455,
        "mean_ms": 22.43080044715447,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          2,
          83,
          37,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 438.212136
      },
      "accepted_interval": {
        "count": 123,
        "sum_ns": 29693485232,
        "mean_ms": 241.4104490406504,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          7,
          10,
          21,
          20,
          32,
          24,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 923.744774
      },
      "completed_queue_wait": {
        "count": 123,
        "sum_ns": 2365873398,
        "mean_ms": 19.234743073170733,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          22,
          26,
          23,
          19,
          9,
          7,
          5,
          6,
          4,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 206.683061
      },
      "preprocess_to_infer_output": {
        "count": 123,
        "sum_ns": 105517689860,
        "mean_ms": 857.8673972357724,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          100,
          22,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1497.723996
      },
      "preprocess_to_callback": {
        "count": 123,
        "sum_ns": 107809044807,
        "mean_ms": 876.4962992439024,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          95,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1515.979255
      },
      "infer_output_to_callback": {
        "count": 123,
        "sum_ns": 2291354947,
        "mean_ms": 18.62890200813008,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          28,
          27,
          41,
          21,
          4,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 564.638836
      },
      "callback_to_copied": {
        "count": 123,
        "sum_ns": 7690832357,
        "mean_ms": 62.527092333333336,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          63,
          27,
          16,
          9,
          2,
          1,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 649.318311
      },
      "selected_to_submitted": {
        "count": 123,
        "sum_ns": 125516469847,
        "mean_ms": 1020.4591044471546,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          62,
          61,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1681.817411
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.130270776804537,
    "first_redis_commit_s": 24.33865022379905,
    "maximum_sampled_silence_ms": 319.936175,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 162,
    "accepted": 162,
    "sent": 57,
    "late": 0,
    "completed_fps": 5.399983372199327,
    "accepted_fps": 5.399983372199327,
    "sent_fps": 1.899994149477541,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.146790622,
    "selected": 163,
    "infer_entries": 162,
    "infer_entry_fps": 5.3737063434466705,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 163,
        "sum_ns": 1867325213,
        "mean_ms": 11.45598290184049,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          31,
          41,
          30,
          34,
          21,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 82.669861
      },
      "preprocess_start_to_preprocess_output": {
        "count": 162,
        "sum_ns": 2456622296,
        "mean_ms": 15.164335160493826,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          2,
          5,
          1,
          38,
          81,
          34,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.732551
      },
      "selected_to_preprocess": {
        "count": 162,
        "sum_ns": 7492003931,
        "mean_ms": 46.246937845679014,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          17,
          100,
          24,
          8,
          8,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 198.068977
      },
      "preprocess_start_to_preprocess": {
        "count": 162,
        "sum_ns": 5644726920,
        "mean_ms": 34.84399333333334,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          3,
          7,
          36,
          93,
          9,
          5,
          6,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 192.429088
      },
      "preprocess_output_to_infer_entry": {
        "count": 162,
        "sum_ns": 3188104624,
        "mean_ms": 19.679658172839506,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          18,
          10,
          16,
          39,
          43,
          20,
          5,
          5,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 175.523264
      },
      "copy_wall": {
        "count": 162,
        "sum_ns": 9349706526,
        "mean_ms": 57.714237814814815,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          80,
          44,
          21,
          11,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 201.690544
      },
      "copy_thread_cpu": {
        "count": 162,
        "sum_ns": 3354669768,
        "mean_ms": 20.707838074074076,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          65,
          97,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 36.453393
      },
      "accepted_interval": {
        "count": 162,
        "sum_ns": 30005294598,
        "mean_ms": 185.2178678888889,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          8,
          13,
          32,
          47,
          45,
          15,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 464.667287
      },
      "completed_queue_wait": {
        "count": 162,
        "sum_ns": 493948662,
        "mean_ms": 3.049065814814815,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          64,
          26,
          39,
          24,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.955546
      },
      "preprocess_to_infer_output": {
        "count": 162,
        "sum_ns": 109764700369,
        "mean_ms": 677.5598788209877,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          13,
          145,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1135.567385
      },
      "preprocess_to_callback": {
        "count": 162,
        "sum_ns": 111860779360,
        "mean_ms": 690.4986380246913,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          146,
          5,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1145.744567
      },
      "infer_output_to_callback": {
        "count": 163,
        "sum_ns": 2109093054,
        "mean_ms": 12.93922119018405,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          26,
          49,
          62,
          24,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.014116
      },
      "callback_to_copied": {
        "count": 162,
        "sum_ns": 9841124596,
        "mean_ms": 60.74768269135802,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          73,
          49,
          22,
          12,
          3,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 212.124277
      },
      "selected_to_submitted": {
        "count": 162,
        "sum_ns": 129403220555,
        "mean_ms": 798.7853120679013,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          144,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1230.079705
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.007288807071745,
    "first_redis_commit_s": 24.208145867101848,
    "maximum_sampled_silence_ms": 289.061745,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 163,
    "accepted": 163,
    "sent": 55,
    "late": 0,
    "completed_fps": 5.4333166028919155,
    "accepted_fps": 5.4333166028919155,
    "sent_fps": 1.8333276880923641,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.980093694,
    "selected": 163,
    "infer_entries": 162,
    "infer_entry_fps": 5.403585514224777,
    "pre_infer_expired": 0,
    "diagnostic_missed": 6,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 162,
        "sum_ns": 1874095919,
        "mean_ms": 11.568493327160494,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          24,
          43,
          27,
          34,
          31,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 57.543674
      },
      "preprocess_start_to_preprocess_output": {
        "count": 161,
        "sum_ns": 2381012121,
        "mean_ms": 14.788895161490682,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          5,
          39,
          82,
          33,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.26995
      },
      "selected_to_preprocess": {
        "count": 161,
        "sum_ns": 7446519434,
        "mean_ms": 46.251673503105586,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          15,
          104,
          20,
          9,
          6,
          3,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 245.175849
      },
      "preprocess_start_to_preprocess": {
        "count": 161,
        "sum_ns": 5610330600,
        "mean_ms": 34.84677391304348,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          1,
          2,
          8,
          49,
          79,
          8,
          8,
          1,
          3,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 242.574748
      },
      "preprocess_output_to_infer_entry": {
        "count": 160,
        "sum_ns": 3203264427,
        "mean_ms": 20.02040266875,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          30,
          6,
          15,
          30,
          44,
          21,
          6,
          3,
          0,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 227.719553
      },
      "copy_wall": {
        "count": 163,
        "sum_ns": 9197084924,
        "mean_ms": 56.42383388957055,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          85,
          41,
          19,
          16,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 149.690517
      },
      "copy_thread_cpu": {
        "count": 163,
        "sum_ns": 3285716272,
        "mean_ms": 20.1577685398773,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          74,
          89,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.512034
      },
      "accepted_interval": {
        "count": 163,
        "sum_ns": 29667081178,
        "mean_ms": 182.00663299386505,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          16,
          39,
          36,
          49,
          10,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 588.889761
      },
      "completed_queue_wait": {
        "count": 163,
        "sum_ns": 411343605,
        "mean_ms": 2.523580398773006,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          64,
          42,
          37,
          15,
          3,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.369491
      },
      "preprocess_to_infer_output": {
        "count": 161,
        "sum_ns": 106967687780,
        "mean_ms": 664.3955762732919,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          141,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1081.541553
      },
      "preprocess_to_callback": {
        "count": 159,
        "sum_ns": 107709018054,
        "mean_ms": 677.4152078867925,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          141,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1092.123565
      },
      "infer_output_to_callback": {
        "count": 160,
        "sum_ns": 2212025467,
        "mean_ms": 13.82515916875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          21,
          47,
          60,
          30,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.27603
      },
      "callback_to_copied": {
        "count": 160,
        "sum_ns": 9611572996,
        "mean_ms": 60.072331225,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          70,
          48,
          22,
          14,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 168.669531
      },
      "selected_to_submitted": {
        "count": 162,
        "sum_ns": 127417648159,
        "mean_ms": 786.5286923395062,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          146,
          14,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1281.416866
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.151647498831153,
    "first_redis_commit_s": 28.528483080677688,
    "maximum_sampled_silence_ms": 339.3696,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 127,
    "accepted": 127,
    "sent": 54,
    "late": 0,
    "completed_fps": 4.233320297958731,
    "accepted_fps": 4.233320297958731,
    "sent_fps": 1.7999944573997757,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.070939345,
    "selected": 127,
    "infer_entries": 127,
    "infer_entry_fps": 4.223346618572351,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 127,
        "sum_ns": 6679412076,
        "mean_ms": 52.59379587401575,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          17,
          58,
          25,
          14,
          8,
          2,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 425.82518
      },
      "copy_thread_cpu": {
        "count": 127,
        "sum_ns": 2294314578,
        "mean_ms": 18.065469118110236,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          14,
          91,
          21,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 317.546108
      },
      "accepted_interval": {
        "count": 127,
        "sum_ns": 29931753200,
        "mean_ms": 235.68309606299212,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          5,
          6,
          20,
          26,
          41,
          20,
          8,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 773.020305
      },
      "completed_queue_wait": {
        "count": 127,
        "sum_ns": 1739811601,
        "mean_ms": 13.69930394488189,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          30,
          19,
          32,
          16,
          9,
          10,
          6,
          2,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 180.595505
      },
      "selected_to_preprocess_start": {
        "count": 126,
        "sum_ns": 1404739994,
        "mean_ms": 11.148730111111112,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          23,
          32,
          21,
          24,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.47281
      },
      "preprocess_start_to_preprocess_output": {
        "count": 126,
        "sum_ns": 2606983869,
        "mean_ms": 20.690348166666666,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          9,
          20,
          49,
          37,
          2,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 392.176968
      },
      "selected_to_preprocess": {
        "count": 126,
        "sum_ns": 7241269318,
        "mean_ms": 57.47039141269841,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          5,
          19,
          54,
          24,
          7,
          8,
          4,
          3,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 394.680272
      },
      "preprocess_start_to_preprocess": {
        "count": 126,
        "sum_ns": 5836529324,
        "mean_ms": 46.3216613015873,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          2,
          7,
          6,
          31,
          49,
          10,
          8,
          6,
          2,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 392.909343
      },
      "preprocess_output_to_infer_entry": {
        "count": 126,
        "sum_ns": 3229545455,
        "mean_ms": 25.631313134920635,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          25,
          13,
          10,
          20,
          23,
          17,
          5,
          5,
          4,
          0,
          4,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 253.115673
      },
      "preprocess_to_infer_output": {
        "count": 127,
        "sum_ns": 105654697015,
        "mean_ms": 831.9267481496064,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          103,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1436.69844
      },
      "preprocess_to_callback": {
        "count": 127,
        "sum_ns": 107135304469,
        "mean_ms": 843.5850745590551,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          99,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1470.553875
      },
      "infer_output_to_callback": {
        "count": 127,
        "sum_ns": 1480607454,
        "mean_ms": 11.658326409448819,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          33,
          41,
          31,
          20,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 91.914937
      },
      "callback_to_copied": {
        "count": 126,
        "sum_ns": 6954220881,
        "mean_ms": 55.19222921428572,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          13,
          53,
          29,
          16,
          10,
          2,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 428.014341
      },
      "selected_to_submitted": {
        "count": 126,
        "sum_ns": 121150870279,
        "mean_ms": 961.5148434841269,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          79,
          47,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1625.144013
      }
    },
    "rejection_reasons": {},
    "first_response_s": 25.304919713642448,
    "first_redis_commit_s": 25.713346655946225,
    "maximum_sampled_silence_ms": 230.240963,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 160,
    "accepted": 160,
    "sent": 58,
    "late": 0,
    "completed_fps": 5.33331691081415,
    "accepted_fps": 5.33331691081415,
    "sent_fps": 1.9333273801701294,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.002948776,
    "selected": 159,
    "infer_entries": 158,
    "infer_entry_fps": 5.266149043536267,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 160,
        "sum_ns": 1434227787,
        "mean_ms": 8.963923668749999,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          35,
          42,
          23,
          39,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.689017
      },
      "preprocess_start_to_preprocess_output": {
        "count": 158,
        "sum_ns": 2609039115,
        "mean_ms": 16.512905791139243,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          3,
          36,
          75,
          41,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.334976
      },
      "selected_to_preprocess": {
        "count": 158,
        "sum_ns": 7161393430,
        "mean_ms": 45.32527487341772,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          26,
          82,
          26,
          13,
          8,
          1,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 304.290941
      },
      "preprocess_start_to_preprocess": {
        "count": 158,
        "sum_ns": 5754562056,
        "mean_ms": 36.42127883544303,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          8,
          42,
          78,
          11,
          11,
          5,
          1,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 301.586532
      },
      "preprocess_output_to_infer_entry": {
        "count": 156,
        "sum_ns": 3137747245,
        "mean_ms": 20.11376439102564,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          24,
          4,
          24,
          29,
          39,
          16,
          12,
          4,
          3,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 289.424417
      },
      "copy_wall": {
        "count": 160,
        "sum_ns": 10629814189,
        "mean_ms": 66.43633868124999,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          49,
          55,
          34,
          18,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 156.787175
      },
      "copy_thread_cpu": {
        "count": 160,
        "sum_ns": 3185688734,
        "mean_ms": 19.9105545875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          71,
          89,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.464111
      },
      "accepted_interval": {
        "count": 160,
        "sum_ns": 30077251684,
        "mean_ms": 187.98282302500002,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          9,
          12,
          31,
          38,
          54,
          15,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 436.464909
      },
      "completed_queue_wait": {
        "count": 160,
        "sum_ns": 453590482,
        "mean_ms": 2.8349405125000002,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          71,
          33,
          28,
          19,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.908063
      },
      "preprocess_to_infer_output": {
        "count": 158,
        "sum_ns": 106180791715,
        "mean_ms": 672.0303273101265,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          147,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1091.907579
      },
      "preprocess_to_callback": {
        "count": 158,
        "sum_ns": 107849884721,
        "mean_ms": 682.5942070949367,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          150,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1094.399709
      },
      "infer_output_to_callback": {
        "count": 159,
        "sum_ns": 1680496167,
        "mean_ms": 10.569158283018869,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          32,
          52,
          56,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.978914
      },
      "callback_to_copied": {
        "count": 160,
        "sum_ns": 10963333414,
        "mean_ms": 68.5208338375,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          47,
          50,
          39,
          21,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 157.818635
      },
      "selected_to_submitted": {
        "count": 160,
        "sum_ns": 127384782115,
        "mean_ms": 796.15488821875,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          146,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1209.633218
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.919169743545353,
    "first_redis_commit_s": 25.33522420283407,
    "maximum_sampled_silence_ms": 229.112065,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 166,
    "accepted": 166,
    "sent": 54,
    "late": 0,
    "completed_fps": 5.533316294969681,
    "accepted_fps": 5.533316294969681,
    "sent_fps": 1.7999944573997757,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.198608087,
    "selected": 166,
    "infer_entries": 166,
    "infer_entry_fps": 5.496942094872917,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 166,
        "sum_ns": 1445446594,
        "mean_ms": 8.707509602409639,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          43,
          43,
          28,
          30,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 37.383841
      },
      "preprocess_start_to_preprocess_output": {
        "count": 165,
        "sum_ns": 2232730871,
        "mean_ms": 13.531702248484848,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          6,
          55,
          74,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.104365
      },
      "selected_to_preprocess": {
        "count": 166,
        "sum_ns": 6531669981,
        "mean_ms": 39.347409524096385,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          2,
          3,
          34,
          92,
          24,
          2,
          4,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 246.946587
      },
      "preprocess_start_to_preprocess": {
        "count": 166,
        "sum_ns": 5086223387,
        "mean_ms": 30.639899921686744,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          2,
          4,
          15,
          54,
          73,
          7,
          2,
          6,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 245.767558
      },
      "preprocess_output_to_infer_entry": {
        "count": 165,
        "sum_ns": 2815362957,
        "mean_ms": 17.0628058,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          36,
          11,
          19,
          35,
          36,
          14,
          5,
          0,
          7,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 236.063368
      },
      "copy_wall": {
        "count": 166,
        "sum_ns": 9071625890,
        "mean_ms": 54.64834873493976,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          80,
          49,
          26,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.909049
      },
      "copy_thread_cpu": {
        "count": 166,
        "sum_ns": 3274492911,
        "mean_ms": 19.725860909638556,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          81,
          85,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.901352
      },
      "accepted_interval": {
        "count": 166,
        "sum_ns": 30136509259,
        "mean_ms": 181.5452365,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          10,
          10,
          37,
          51,
          45,
          11,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 536.916861
      },
      "completed_queue_wait": {
        "count": 166,
        "sum_ns": 479734229,
        "mean_ms": 2.8899652349397593,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          67,
          32,
          38,
          21,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.096657
      },
      "preprocess_to_infer_output": {
        "count": 167,
        "sum_ns": 108726438741,
        "mean_ms": 651.0565194071856,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          23,
          140,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1065.000064
      },
      "preprocess_to_callback": {
        "count": 167,
        "sum_ns": 110500835786,
        "mean_ms": 661.6816514131737,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          17,
          146,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1080.547264
      },
      "infer_output_to_callback": {
        "count": 167,
        "sum_ns": 1774397045,
        "mean_ms": 10.625132005988025,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          38,
          56,
          59,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.014585
      },
      "callback_to_copied": {
        "count": 166,
        "sum_ns": 9521916756,
        "mean_ms": 57.360944313253015,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          77,
          47,
          29,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 152.516268
      },
      "selected_to_submitted": {
        "count": 166,
        "sum_ns": 126073534292,
        "mean_ms": 759.4791222409638,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          152,
          9,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1247.302942
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.592316841706634,
    "first_redis_commit_s": 27.932731347624213,
    "maximum_sampled_silence_ms": 260.988596,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 130,
    "accepted": 130,
    "sent": 55,
    "late": 0,
    "completed_fps": 4.333319990036498,
    "accepted_fps": 4.333319990036498,
    "sent_fps": 1.8333276880923641,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.435655073,
    "selected": 130,
    "infer_entries": 129,
    "infer_entry_fps": 4.238449926265532,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 130,
        "sum_ns": 1403388099,
        "mean_ms": 10.79529306923077,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          22,
          37,
          25,
          25,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 69.323766
      },
      "copy_wall": {
        "count": 130,
        "sum_ns": 6698406974,
        "mean_ms": 51.52620749230769,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          28,
          53,
          28,
          11,
          5,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 517.867924
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2331212249,
        "mean_ms": 17.932401915384617,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          23,
          99,
          6,
          0,
          0,
          0,
          0,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 355.694117
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 30534971003,
        "mean_ms": 234.88439233076923,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          4,
          9,
          16,
          26,
          47,
          18,
          8,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 680.425364
      },
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 1620140515,
        "mean_ms": 12.462619346153845,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          31,
          25,
          26,
          25,
          5,
          9,
          2,
          2,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.890094
      },
      "preprocess_start_to_preprocess_output": {
        "count": 128,
        "sum_ns": 1955811872,
        "mean_ms": 15.27978025,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          8,
          38,
          43,
          34,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.222169
      },
      "selected_to_preprocess": {
        "count": 129,
        "sum_ns": 7588815274,
        "mean_ms": 58.828025379844966,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          3,
          23,
          55,
          19,
          7,
          13,
          4,
          3,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 317.408434
      },
      "preprocess_start_to_preprocess": {
        "count": 129,
        "sum_ns": 6191980608,
        "mean_ms": 47.999849674418606,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          2,
          0,
          12,
          31,
          51,
          9,
          5,
          12,
          2,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 315.04312
      },
      "preprocess_output_to_infer_entry": {
        "count": 127,
        "sum_ns": 3897981788,
        "mean_ms": 30.692769984251967,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          22,
          2,
          20,
          22,
          28,
          7,
          7,
          7,
          8,
          0,
          4,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 283.056742
      },
      "preprocess_to_infer_output": {
        "count": 129,
        "sum_ns": 106081607983,
        "mean_ms": 822.3380463798449,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          112,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1332.732061
      },
      "preprocess_to_callback": {
        "count": 128,
        "sum_ns": 106756907770,
        "mean_ms": 834.038341953125,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          106,
          17,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1336.691541
      },
      "infer_output_to_callback": {
        "count": 129,
        "sum_ns": 1528529324,
        "mean_ms": 11.849064527131782,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          20,
          46,
          45,
          12,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 90.965886
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 6933016707,
        "mean_ms": 53.330897746153845,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          24,
          54,
          29,
          12,
          6,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 520.455679
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 123846675053,
        "mean_ms": 952.666731176923,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          83,
          47,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1461.605507
      }
    },
    "rejection_reasons": {},
    "first_response_s": 28.416096594184637,
    "first_redis_commit_s": 28.96057755406946,
    "maximum_sampled_silence_ms": 538.05403,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 163,
    "accepted": 163,
    "sent": 61,
    "late": 0,
    "completed_fps": 5.4333166028919155,
    "accepted_fps": 5.4333166028919155,
    "sent_fps": 2.0333270722478947,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.63899294,
    "selected": 163,
    "infer_entries": 163,
    "infer_entry_fps": 5.32001819769994,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "copy_wall": {
        "count": 163,
        "sum_ns": 9610160000,
        "mean_ms": 58.95803680981595,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          66,
          54,
          28,
          9,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 163.872397
      },
      "copy_thread_cpu": {
        "count": 163,
        "sum_ns": 3245815315,
        "mean_ms": 19.912977392638034,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          81,
          82,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.472757
      },
      "accepted_interval": {
        "count": 163,
        "sum_ns": 30591278079,
        "mean_ms": 187.67655263190184,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          6,
          11,
          38,
          47,
          45,
          13,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 557.136027
      },
      "selected_to_preprocess_start": {
        "count": 163,
        "sum_ns": 1739607692,
        "mean_ms": 10.67243982822086,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          39,
          34,
          30,
          30,
          28,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.730303
      },
      "completed_queue_wait": {
        "count": 163,
        "sum_ns": 498084296,
        "mean_ms": 3.0557318773006137,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          59,
          37,
          43,
          13,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.752467
      },
      "preprocess_start_to_preprocess_output": {
        "count": 162,
        "sum_ns": 2536520533,
        "mean_ms": 15.657534154320986,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          9,
          39,
          65,
          46,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.37832
      },
      "selected_to_preprocess": {
        "count": 163,
        "sum_ns": 7319411496,
        "mean_ms": 44.904365006134974,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          3,
          18,
          92,
          30,
          10,
          5,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 215.102056
      },
      "preprocess_start_to_preprocess": {
        "count": 163,
        "sum_ns": 5579803804,
        "mean_ms": 34.231925177914114,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          4,
          12,
          39,
          82,
          15,
          3,
          4,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 201.171956
      },
      "preprocess_output_to_infer_entry": {
        "count": 162,
        "sum_ns": 3000153100,
        "mean_ms": 18.519463580246914,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          23,
          10,
          16,
          31,
          49,
          19,
          5,
          3,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.365734
      },
      "preprocess_to_infer_output": {
        "count": 162,
        "sum_ns": 108549486409,
        "mean_ms": 670.0585580802469,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          15,
          144,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1104.649647
      },
      "preprocess_to_callback": {
        "count": 163,
        "sum_ns": 110684492198,
        "mean_ms": 679.045964404908,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          11,
          149,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1108.88052
      },
      "infer_output_to_callback": {
        "count": 162,
        "sum_ns": 1525380403,
        "mean_ms": 9.415928413580247,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          45,
          58,
          43,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.863717
      },
      "callback_to_copied": {
        "count": 163,
        "sum_ns": 10154738869,
        "mean_ms": 62.29901146625767,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          55,
          61,
          29,
          12,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 164.694542
      },
      "selected_to_submitted": {
        "count": 163,
        "sum_ns": 128274337836,
        "mean_ms": 786.9591278282209,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          147,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1157.077937
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.65567419398576,
    "first_redis_commit_s": 27.885421512182802,
    "maximum_sampled_silence_ms": 84.634163,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 167,
    "accepted": 167,
    "sent": 58,
    "late": 0,
    "completed_fps": 5.566649525662269,
    "accepted_fps": 5.566649525662269,
    "sent_fps": 1.9333273801701294,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.446544265,
    "selected": 167,
    "infer_entries": 167,
    "infer_entry_fps": 5.485023145696565,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "accepted_interval": {
        "count": 167,
        "sum_ns": 30432251875,
        "mean_ms": 182.22905314371258,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          9,
          19,
          43,
          36,
          43,
          13,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 713.333768
      },
      "completed_queue_wait": {
        "count": 167,
        "sum_ns": 547098428,
        "mean_ms": 3.2760384910179643,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          67,
          29,
          38,
          20,
          10,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.720759
      },
      "copy_wall": {
        "count": 167,
        "sum_ns": 9703175710,
        "mean_ms": 58.10284856287425,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          70,
          51,
          23,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 178.620214
      },
      "copy_thread_cpu": {
        "count": 167,
        "sum_ns": 3266274102,
        "mean_ms": 19.558527556886226,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          86,
          81,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.328989
      },
      "selected_to_preprocess_start": {
        "count": 167,
        "sum_ns": 1312054628,
        "mean_ms": 7.856614538922155,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          43,
          48,
          28,
          31,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.607674
      },
      "preprocess_start_to_preprocess_output": {
        "count": 167,
        "sum_ns": 2486528615,
        "mean_ms": 14.889392904191617,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          6,
          33,
          93,
          32,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.188152
      },
      "selected_to_preprocess": {
        "count": 167,
        "sum_ns": 6735378163,
        "mean_ms": 40.331605766467064,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          2,
          33,
          100,
          15,
          7,
          5,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.799725
      },
      "preprocess_start_to_preprocess": {
        "count": 167,
        "sum_ns": 5423323535,
        "mean_ms": 32.47499122754491,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          1,
          2,
          6,
          56,
          79,
          11,
          4,
          4,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 190.616619
      },
      "preprocess_output_to_infer_entry": {
        "count": 167,
        "sum_ns": 2936794920,
        "mean_ms": 17.585598323353295,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          28,
          5,
          20,
          39,
          48,
          12,
          5,
          4,
          3,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 170.872905
      },
      "preprocess_to_infer_output": {
        "count": 166,
        "sum_ns": 108655701557,
        "mean_ms": 654.5524190180722,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          31,
          133,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1048.697233
      },
      "preprocess_to_callback": {
        "count": 166,
        "sum_ns": 110325114560,
        "mean_ms": 664.6091238554217,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          24,
          139,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1052.179325
      },
      "infer_output_to_callback": {
        "count": 165,
        "sum_ns": 1723731689,
        "mean_ms": 10.44685872121212,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          40,
          63,
          43,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.073976
      },
      "callback_to_copied": {
        "count": 165,
        "sum_ns": 9955364338,
        "mean_ms": 60.335541442424244,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          64,
          53,
          25,
          18,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 179.22033
      },
      "selected_to_submitted": {
        "count": 166,
        "sum_ns": 126935760518,
        "mean_ms": 764.6732561325301,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          145,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1205.083604
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.88327840389684,
    "first_redis_commit_s": 31.71200527716428,
    "maximum_sampled_silence_ms": 179.780151,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 134,
    "accepted": 134,
    "sent": 59,
    "late": 0,
    "completed_fps": 4.466652912806851,
    "accepted_fps": 4.466652912806851,
    "sent_fps": 1.966660610862718,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.449503837,
    "selected": 134,
    "infer_entries": 133,
    "infer_entry_fps": 4.367887263843957,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 134,
        "sum_ns": 5479242688,
        "mean_ms": 40.88987080597015,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          3,
          25,
          63,
          32,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 124.83257
      },
      "copy_thread_cpu": {
        "count": 134,
        "sum_ns": 1700280110,
        "mean_ms": 12.688657537313434,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          35,
          91,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.968537
      },
      "accepted_interval": {
        "count": 134,
        "sum_ns": 30430446437,
        "mean_ms": 227.09288385820895,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          5,
          5,
          19,
          33,
          40,
          25,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 738.916837
      },
      "selected_to_preprocess_start": {
        "count": 134,
        "sum_ns": 1149068024,
        "mean_ms": 8.575134507462685,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          36,
          38,
          24,
          18,
          16,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 70.606506
      },
      "completed_queue_wait": {
        "count": 134,
        "sum_ns": 880985060,
        "mean_ms": 6.574515373134328,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          39,
          28,
          27,
          17,
          14,
          5,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 85.209695
      },
      "preprocess_start_to_preprocess_output": {
        "count": 133,
        "sum_ns": 2043636713,
        "mean_ms": 15.365689571428572,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          15,
          25,
          62,
          26,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 88.644252
      },
      "selected_to_preprocess": {
        "count": 133,
        "sum_ns": 6305218047,
        "mean_ms": 47.4076544887218,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          7,
          22,
          65,
          18,
          9,
          7,
          2,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 305.465589
      },
      "preprocess_start_to_preprocess": {
        "count": 133,
        "sum_ns": 5163114746,
        "mean_ms": 38.820411624060156,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          6,
          12,
          35,
          53,
          10,
          5,
          8,
          1,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 303.240637
      },
      "preprocess_output_to_infer_entry": {
        "count": 132,
        "sum_ns": 3112007304,
        "mean_ms": 23.57581290909091,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          30,
          7,
          16,
          25,
          18,
          15,
          10,
          3,
          5,
          1,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 300.590559
      },
      "preprocess_to_infer_output": {
        "count": 130,
        "sum_ns": 106592891337,
        "mean_ms": 819.9453179769231,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          113,
          16,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1371.260282
      },
      "preprocess_to_callback": {
        "count": 130,
        "sum_ns": 108057767667,
        "mean_ms": 831.2135974384615,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          112,
          17,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1376.888927
      },
      "infer_output_to_callback": {
        "count": 130,
        "sum_ns": 1464876330,
        "mean_ms": 11.268279461538462,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          32,
          37,
          43,
          16,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.511155
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 5570442509,
        "mean_ms": 42.84955776153846,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          2,
          23,
          59,
          33,
          10,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 129.433654
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 120210958962,
        "mean_ms": 924.699684323077,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          87,
          42,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1477.0745
      }
    },
    "rejection_reasons": {},
    "first_response_s": 29.695533701684326,
    "first_redis_commit_s": 30.891920868773013,
    "maximum_sampled_silence_ms": 177.203228,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 161,
    "accepted": 161,
    "sent": 58,
    "late": 0,
    "completed_fps": 5.366650141506739,
    "accepted_fps": 5.366650141506739,
    "sent_fps": 1.9333273801701294,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.411883074,
    "selected": 160,
    "infer_entries": 160,
    "infer_entry_fps": 5.261101379703404,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "completed_queue_wait": {
        "count": 162,
        "sum_ns": 488039359,
        "mean_ms": 3.0125886358024694,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          59,
          27,
          46,
          23,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.740116
      },
      "selected_to_preprocess_start": {
        "count": 160,
        "sum_ns": 1404098207,
        "mean_ms": 8.775613793749999,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          42,
          44,
          23,
          31,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.701953
      },
      "preprocess_start_to_preprocess_output": {
        "count": 159,
        "sum_ns": 2261472439,
        "mean_ms": 14.223097100628932,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          4,
          4,
          9,
          38,
          68,
          36,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.864249
      },
      "selected_to_preprocess": {
        "count": 159,
        "sum_ns": 6118685719,
        "mean_ms": 38.482300119496855,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          2,
          36,
          85,
          22,
          7,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 175.938412
      },
      "preprocess_start_to_preprocess": {
        "count": 159,
        "sum_ns": 4716514838,
        "mean_ms": 29.663615333333333,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          1,
          6,
          13,
          48,
          72,
          9,
          4,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 162.49985
      },
      "preprocess_output_to_infer_entry": {
        "count": 158,
        "sum_ns": 2418377859,
        "mean_ms": 15.306188981012658,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          31,
          7,
          26,
          32,
          41,
          10,
          2,
          3,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 155.133144
      },
      "copy_wall": {
        "count": 161,
        "sum_ns": 9297574473,
        "mean_ms": 57.74890977018634,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          74,
          47,
          23,
          12,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 168.304323
      },
      "copy_thread_cpu": {
        "count": 161,
        "sum_ns": 3161975543,
        "mean_ms": 19.639599645962733,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          83,
          78,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.554271
      },
      "accepted_interval": {
        "count": 161,
        "sum_ns": 30408036714,
        "mean_ms": 188.86979325465836,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          5,
          12,
          33,
          44,
          52,
          13,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 573.125975
      },
      "preprocess_to_infer_output": {
        "count": 156,
        "sum_ns": 106258450301,
        "mean_ms": 681.1439121858974,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          146,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1048.896434
      },
      "preprocess_to_callback": {
        "count": 156,
        "sum_ns": 107941117802,
        "mean_ms": 691.9302423205128,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          149,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1061.43683
      },
      "infer_output_to_callback": {
        "count": 157,
        "sum_ns": 1687708717,
        "mean_ms": 10.749737050955416,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          28,
          73,
          38,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 51.620611
      },
      "callback_to_copied": {
        "count": 157,
        "sum_ns": 9417108999,
        "mean_ms": 59.98158598089172,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          67,
          49,
          26,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 171.582571
      },
      "selected_to_submitted": {
        "count": 157,
        "sum_ns": 124079074486,
        "mean_ms": 790.3125763439491,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          147,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1169.105703
      }
    },
    "rejection_reasons": {},
    "first_response_s": 28.979734705761075,
    "first_redis_commit_s": 30.49013399006799,
    "maximum_sampled_silence_ms": 184.546166,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000092376954854,
    "completed": 166,
    "accepted": 166,
    "sent": 59,
    "late": 0,
    "completed_fps": 5.533316294969681,
    "accepted_fps": 5.533316294969681,
    "sent_fps": 1.966660610862718,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 30.15584362,
    "selected": 166,
    "infer_entries": 164,
    "infer_entry_fps": 5.438415255981488,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 164,
        "sum_ns": 1222067375,
        "mean_ms": 7.451630335365853,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          40,
          51,
          32,
          30,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.339881
      },
      "preprocess_start_to_preprocess_output": {
        "count": 164,
        "sum_ns": 2043345640,
        "mean_ms": 12.459424634146343,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          8,
          22,
          37,
          75,
          21,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.794359
      },
      "selected_to_preprocess": {
        "count": 163,
        "sum_ns": 6098288083,
        "mean_ms": 37.41281032515337,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          6,
          36,
          92,
          14,
          6,
          4,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 194.585461
      },
      "preprocess_start_to_preprocess": {
        "count": 163,
        "sum_ns": 4903175672,
        "mean_ms": 30.08083234355828,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          6,
          17,
          51,
          68,
          7,
          4,
          5,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 183.824014
      },
      "preprocess_output_to_infer_entry": {
        "count": 163,
        "sum_ns": 2866143149,
        "mean_ms": 17.583700300613497,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          33,
          7,
          18,
          34,
          43,
          12,
          8,
          0,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 174.225631
      },
      "copy_wall": {
        "count": 165,
        "sum_ns": 8962296450,
        "mean_ms": 54.31694818181818,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          83,
          44,
          25,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.566213
      },
      "copy_thread_cpu": {
        "count": 165,
        "sum_ns": 3268651497,
        "mean_ms": 19.810009072727276,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          90,
          75,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.92493
      },
      "accepted_interval": {
        "count": 165,
        "sum_ns": 30141567055,
        "mean_ms": 182.67616396969697,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          12,
          11,
          46,
          32,
          41,
          16,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 558.529779
      },
      "completed_queue_wait": {
        "count": 165,
        "sum_ns": 454095242,
        "mean_ms": 2.752092375757576,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          69,
          27,
          42,
          20,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.055268
      },
      "preprocess_to_infer_output": {
        "count": 159,
        "sum_ns": 103982530713,
        "mean_ms": 653.9781805849057,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          138,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1052.895899
      },
      "preprocess_to_callback": {
        "count": 159,
        "sum_ns": 105815154418,
        "mean_ms": 665.504115836478,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          18,
          139,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1062.845132
      },
      "infer_output_to_callback": {
        "count": 160,
        "sum_ns": 1835777748,
        "mean_ms": 11.473610925000001,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          35,
          46,
          61,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.405244
      },
      "callback_to_copied": {
        "count": 160,
        "sum_ns": 9178782283,
        "mean_ms": 57.367389268749996,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          75,
          45,
          25,
          7,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 186.780288
      },
      "selected_to_submitted": {
        "count": 160,
        "sum_ns": 122295674921,
        "mean_ms": 764.34796825625,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          145,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1258.172158
      }
    },
    "rejection_reasons": {},
    "first_response_s": 28.84793023392558,
    "first_redis_commit_s": 35.19613285083324,
    "maximum_sampled_silence_ms": 179.262402,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 125,
    "accepted": 125,
    "sent": 55,
    "late": 0,
    "completed_fps": 4.166653836573555,
    "accepted_fps": 4.166653836573555,
    "sent_fps": 1.8333276880923641,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.418699325,
    "selected": 126,
    "infer_entries": 127,
    "infer_entry_fps": 4.175063458272965,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "completed_queue_wait": {
        "count": 127,
        "sum_ns": 3056808072,
        "mean_ms": 24.069354897637794,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          29,
          29,
          30,
          13,
          5,
          3,
          5,
          1,
          4,
          3,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 322.754746
      },
      "selected_to_preprocess_start": {
        "count": 124,
        "sum_ns": 1690802211,
        "mean_ms": 13.635501701612903,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          28,
          43,
          7,
          18,
          22,
          3,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 124.710777
      },
      "preprocess_start_to_preprocess_output": {
        "count": 122,
        "sum_ns": 3213472886,
        "mean_ms": 26.33994168852459,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          1,
          1,
          2,
          25,
          51,
          31,
          6,
          3,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 653.465197
      },
      "copy_wall": {
        "count": 125,
        "sum_ns": 8383953427,
        "mean_ms": 67.071627416,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          10,
          66,
          19,
          15,
          10,
          1,
          0,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 855.305554
      },
      "copy_thread_cpu": {
        "count": 125,
        "sum_ns": 3708473469,
        "mean_ms": 29.667787752,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          7,
          83,
          32,
          0,
          0,
          0,
          0,
          0,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 543.707644
      },
      "accepted_interval": {
        "count": 125,
        "sum_ns": 30190743043,
        "mean_ms": 241.525944344,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          5,
          6,
          20,
          25,
          38,
          19,
          8,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 954.775635
      },
      "selected_to_preprocess": {
        "count": 126,
        "sum_ns": 9748813293,
        "mean_ms": 77.37153407142857,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          59,
          19,
          13,
          11,
          6,
          4,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 694.591656
      },
      "preprocess_start_to_preprocess": {
        "count": 125,
        "sum_ns": 7975916337,
        "mean_ms": 63.807330696,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          1,
          0,
          4,
          21,
          58,
          12,
          10,
          7,
          5,
          4,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 693.115561
      },
      "preprocess_output_to_infer_entry": {
        "count": 124,
        "sum_ns": 4690450629,
        "mean_ms": 37.82621475,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          16,
          6,
          12,
          22,
          27,
          20,
          4,
          6,
          3,
          3,
          4,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 670.090723
      },
      "preprocess_to_infer_output": {
        "count": 125,
        "sum_ns": 105757013600,
        "mean_ms": 846.0561088,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          100,
          25,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1396.587672
      },
      "preprocess_to_callback": {
        "count": 125,
        "sum_ns": 107429378658,
        "mean_ms": 859.435029264,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          98,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1408.938168
      },
      "infer_output_to_callback": {
        "count": 125,
        "sum_ns": 1672365058,
        "mean_ms": 13.378920464,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          20,
          45,
          38,
          16,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 98.395672
      },
      "callback_to_copied": {
        "count": 124,
        "sum_ns": 8579226238,
        "mean_ms": 69.18730837096774,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          7,
          63,
          24,
          14,
          11,
          1,
          0,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 855.875599
      },
      "selected_to_submitted": {
        "count": 124,
        "sum_ns": 125282087054,
        "mean_ms": 1010.3394117258065,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          65,
          59,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1723.422005
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.37550678802654,
    "first_redis_commit_s": 21.44207639945671,
    "maximum_sampled_silence_ms": 305.850034,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 164,
    "accepted": 164,
    "sent": 57,
    "late": 0,
    "completed_fps": 5.466649833584504,
    "accepted_fps": 5.466649833584504,
    "sent_fps": 1.899994149477541,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.567797289,
    "selected": 164,
    "infer_entries": 164,
    "infer_entry_fps": 5.36512325207732,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 164,
        "sum_ns": 1579235972,
        "mean_ms": 9.629487634146342,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          25,
          48,
          28,
          45,
          17,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.491405
      },
      "preprocess_start_to_preprocess_output": {
        "count": 164,
        "sum_ns": 2464411692,
        "mean_ms": 15.026900560975609,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          2,
          37,
          89,
          33,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.684936
      },
      "selected_to_preprocess": {
        "count": 164,
        "sum_ns": 7041515515,
        "mean_ms": 42.93607021341463,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          24,
          96,
          28,
          6,
          4,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 277.723093
      },
      "preprocess_start_to_preprocess": {
        "count": 164,
        "sum_ns": 5462279543,
        "mean_ms": 33.306582579268294,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          1,
          2,
          8,
          55,
          73,
          16,
          4,
          1,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 275.727341
      },
      "preprocess_output_to_infer_entry": {
        "count": 164,
        "sum_ns": 2997867851,
        "mean_ms": 18.279682018292686,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          34,
          5,
          13,
          35,
          40,
          24,
          6,
          2,
          1,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 258.500341
      },
      "copy_wall": {
        "count": 164,
        "sum_ns": 9759595114,
        "mean_ms": 59.50972630487805,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          71,
          48,
          24,
          13,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 169.225091
      },
      "copy_thread_cpu": {
        "count": 164,
        "sum_ns": 3328272913,
        "mean_ms": 20.294347030487806,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          71,
          93,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.682531
      },
      "accepted_interval": {
        "count": 164,
        "sum_ns": 30419189659,
        "mean_ms": 185.48286377439024,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          10,
          16,
          34,
          40,
          46,
          14,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 507.789538
      },
      "completed_queue_wait": {
        "count": 164,
        "sum_ns": 423937101,
        "mean_ms": 2.5849823231707316,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          70,
          36,
          36,
          15,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.119094
      },
      "preprocess_to_infer_output": {
        "count": 163,
        "sum_ns": 111129953852,
        "mean_ms": 681.77885798773,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          146,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1027.047067
      },
      "preprocess_to_callback": {
        "count": 163,
        "sum_ns": 112819577168,
        "mean_ms": 692.144645202454,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          146,
          5,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1039.93487
      },
      "infer_output_to_callback": {
        "count": 164,
        "sum_ns": 1707538387,
        "mean_ms": 10.41181943292683,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          30,
          63,
          50,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.270472
      },
      "callback_to_copied": {
        "count": 164,
        "sum_ns": 10102397484,
        "mean_ms": 61.599984658536584,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          68,
          49,
          24,
          15,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 170.778023
      },
      "selected_to_submitted": {
        "count": 164,
        "sum_ns": 130848324143,
        "mean_ms": 797.8556350182927,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          147,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1138.296907
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.20394528610632,
    "first_redis_commit_s": 21.238167728297412,
    "maximum_sampled_silence_ms": 310.079308,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000092376954854,
    "completed": 166,
    "accepted": 166,
    "sent": 54,
    "late": 0,
    "completed_fps": 5.533316294969681,
    "accepted_fps": 5.533316294969681,
    "sent_fps": 1.7999944573997757,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.28589681,
    "selected": 167,
    "infer_entries": 166,
    "infer_entry_fps": 5.481099042283899,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 165,
        "sum_ns": 2189920139,
        "mean_ms": 13.272243266666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          23,
          41,
          34,
          38,
          26,
          2,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 337.205968
      },
      "preprocess_start_to_preprocess_output": {
        "count": 165,
        "sum_ns": 2520560982,
        "mean_ms": 15.276127163636364,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          3,
          36,
          87,
          37,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.21882
      },
      "selected_to_preprocess": {
        "count": 164,
        "sum_ns": 7901085197,
        "mean_ms": 48.17734876219512,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          15,
          98,
          33,
          6,
          4,
          3,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 362.269677
      },
      "preprocess_start_to_preprocess": {
        "count": 164,
        "sum_ns": 5712826579,
        "mean_ms": 34.834308408536586,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          6,
          42,
          96,
          9,
          3,
          4,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 247.024944
      },
      "preprocess_output_to_infer_entry": {
        "count": 164,
        "sum_ns": 3212264790,
        "mean_ms": 19.586980426829268,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          25,
          6,
          7,
          51,
          43,
          20,
          4,
          1,
          3,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 233.587102
      },
      "copy_wall": {
        "count": 166,
        "sum_ns": 10332715424,
        "mean_ms": 62.24527363855422,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          78,
          45,
          23,
          18,
          1,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 376.407072
      },
      "copy_thread_cpu": {
        "count": 166,
        "sum_ns": 3438215824,
        "mean_ms": 20.71214351807229,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          68,
          98,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.03609
      },
      "accepted_interval": {
        "count": 166,
        "sum_ns": 30147423667,
        "mean_ms": 181.61098594578314,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          12,
          15,
          41,
          36,
          38,
          19,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 496.59563
      },
      "completed_queue_wait": {
        "count": 166,
        "sum_ns": 400637182,
        "mean_ms": 2.413477,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          87,
          23,
          31,
          18,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.903121
      },
      "preprocess_to_infer_output": {
        "count": 165,
        "sum_ns": 108302323357,
        "mean_ms": 656.3777173151516,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          146,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 987.952725
      },
      "preprocess_to_callback": {
        "count": 164,
        "sum_ns": 109798251065,
        "mean_ms": 669.5015308841463,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          148,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 998.001285
      },
      "infer_output_to_callback": {
        "count": 164,
        "sum_ns": 2032313579,
        "mean_ms": 12.392155969512194,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          27,
          51,
          58,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 49.359942
      },
      "callback_to_copied": {
        "count": 163,
        "sum_ns": 10559615641,
        "mean_ms": 64.78291804294479,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          68,
          50,
          21,
          22,
          1,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 377.270874
      },
      "selected_to_submitted": {
        "count": 164,
        "sum_ns": 128359205111,
        "mean_ms": 782.678079945122,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          151,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1200.178638
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.152840240392834,
    "first_redis_commit_s": 22.59020359022543,
    "maximum_sampled_silence_ms": 179.622438,
    "redis_commit_p95_bucket_upper_ms": 2000
  }
]