[
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000055365730077,
    "completed": 296,
    "accepted": 296,
    "sent": 280,
    "late": 0,
    "completed_fps": 9.866648457526825,
    "accepted_fps": 9.866648457526825,
    "sent_fps": 9.33331610847132,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.749820318,
    "selected": 297,
    "infer_entries": 297,
    "infer_entry_fps": 9.983253573477935,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 2238791601,
        "mean_ms": 7.563485138513514,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          35,
          118,
          67,
          51,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.086828
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 1912404829,
        "mean_ms": 6.460827125,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          29,
          64,
          66,
          81,
          40,
          15,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 56.800208
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 5623910960,
        "mean_ms": 18.999699189189187,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          44,
          70,
          71,
          92,
          17,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 93.892833
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 3385119359,
        "mean_ms": 11.436214050675675,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          47,
          76,
          49,
          78,
          41,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 88.137588
      },
      "preprocess_output_to_infer_entry": {
        "count": 296,
        "sum_ns": 1472714530,
        "mean_ms": 4.975386925675675,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          127,
          34,
          43,
          53,
          24,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.846489
      },
      "preprocess_to_infer_output": {
        "count": 296,
        "sum_ns": 17933312518,
        "mean_ms": 60.585515263513514,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          165,
          72,
          22,
          28,
          4,
          1,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 716.261543
      },
      "preprocess_to_callback": {
        "count": 296,
        "sum_ns": 20348038811,
        "mean_ms": 68.74337436148647,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          143,
          82,
          30,
          30,
          7,
          1,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 811.838566
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 2414726293,
        "mean_ms": 8.157859097972974,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          40,
          134,
          82,
          34,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 757.443114
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 6743088092,
        "mean_ms": 22.780703013513513,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          4,
          69,
          112,
          104,
          4,
          1,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 688.556467
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5495265123,
        "mean_ms": 18.565084875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          6,
          89,
          140,
          58,
          1,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 666.558383
      },
      "callback_to_copied": {
        "count": 295,
        "sum_ns": 7027978297,
        "mean_ms": 23.823655244067794,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          60,
          112,
          113,
          6,
          1,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 689.237657
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29660778442,
        "mean_ms": 100.20533257432433,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          16,
          58,
          54,
          67,
          77,
          12,
          2,
          1,
          9,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 923.863556
      },
      "selected_to_submitted": {
        "count": 295,
        "sum_ns": 33301972084,
        "mean_ms": 112.88804096271187,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          14,
          88,
          74,
          70,
          29,
          14,
          1,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 876.420771
      },
      "completed_queue_wait": {
        "count": 281,
        "sum_ns": 56839783928,
        "mean_ms": 202.2768111316726,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          50,
          26,
          25,
          7,
          4,
          13,
          8,
          6,
          17,
          19,
          38,
          29,
          39,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 999.621451
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.134857448283583,
    "first_redis_commit_s": 22.24085191823542,
    "maximum_sampled_silence_ms": 121.755293,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000055365730077,
    "completed": 298,
    "accepted": 298,
    "sent": 299,
    "late": 0,
    "completed_fps": 9.933315001158762,
    "accepted_fps": 9.933315001158762,
    "sent_fps": 9.966648272974732,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.889840467,
    "selected": 299,
    "infer_entries": 299,
    "infer_entry_fps": 10.003398992045883,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 299,
        "sum_ns": 1064509352,
        "mean_ms": 3.5602319464882943,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          66,
          195,
          29,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.857381
      },
      "preprocess_start_to_preprocess_output": {
        "count": 299,
        "sum_ns": 639453680,
        "mean_ms": 2.138641070234114,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          61,
          154,
          56,
          24,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.864118
      },
      "selected_to_preprocess": {
        "count": 299,
        "sum_ns": 1943353819,
        "mean_ms": 6.4995111003344475,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          134,
          130,
          27,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.121918
      },
      "preprocess_start_to_preprocess": {
        "count": 299,
        "sum_ns": 878844467,
        "mean_ms": 2.939279153846154,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          7,
          144,
          108,
          31,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 19.024272
      },
      "preprocess_output_to_infer_entry": {
        "count": 299,
        "sum_ns": 239390787,
        "mean_ms": 0.8006380836120401,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          264,
          18,
          12,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 7.936156
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 9244253792,
        "mean_ms": 30.917236762541805,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          23,
          264,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.051222
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 10412213829,
        "mean_ms": 34.823457622073576,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          267,
          26,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.529069
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 1167960037,
        "mean_ms": 3.9062208595317727,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          32,
          204,
          58,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.17406
      },
      "copy_wall": {
        "count": 298,
        "sum_ns": 6208595407,
        "mean_ms": 20.834212775167785,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          146,
          152,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.201282
      },
      "copy_thread_cpu": {
        "count": 298,
        "sum_ns": 5681685315,
        "mean_ms": 19.066058104026848,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          184,
          114,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.199496
      },
      "callback_to_copied": {
        "count": 298,
        "sum_ns": 6578962646,
        "mean_ms": 22.077055859060405,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          124,
          174,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 43.678333
      },
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29804035894,
        "mean_ms": 100.01354326845637,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          47,
          99,
          137,
          9,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 246.394111
      },
      "selected_to_submitted": {
        "count": 298,
        "sum_ns": 19116642920,
        "mean_ms": 64.14980845637584,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          44,
          195,
          52,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 130.402552
      },
      "completed_queue_wait": {
        "count": 298,
        "sum_ns": 380011270,
        "mean_ms": 1.2752056040268456,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          167,
          90,
          37,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 9.459046
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.192372483201325,
    "first_redis_commit_s": 22.237222412135452,
    "maximum_sampled_silence_ms": 87.271899,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000055365730077,
    "completed": 301,
    "accepted": 301,
    "sent": 300,
    "late": 0,
    "completed_fps": 10.03331481660667,
    "accepted_fps": 10.03331481660667,
    "sent_fps": 9.9999815447907,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.798945983,
    "selected": 300,
    "infer_entries": 297,
    "infer_entry_fps": 9.966795475565998,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "completed_queue_wait": {
        "count": 299,
        "sum_ns": 330572602,
        "mean_ms": 1.1055939866220736,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          177,
          96,
          24,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 7.173145
      },
      "selected_to_preprocess_start": {
        "count": 297,
        "sum_ns": 994666461,
        "mean_ms": 3.3490453232323234,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          55,
          206,
          30,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.20411
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 610129520,
        "mean_ms": 2.0612483783783784,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          70,
          139,
          65,
          19,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.337829
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 1878740549,
        "mean_ms": 6.3257257542087535,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          1,
          151,
          105,
          35,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.377724
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 875903693,
        "mean_ms": 2.959134097972973,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          2,
          159,
          99,
          24,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 19.173614
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 275772968,
        "mean_ms": 0.9285285117845118,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          256,
          21,
          11,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.188222
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 9134865746,
        "mean_ms": 30.757123723905725,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          22,
          261,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 58.59902
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 10311020441,
        "mean_ms": 34.71724054208754,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          272,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.061974
      },
      "infer_output_to_callback": {
        "count": 298,
        "sum_ns": 1179439230,
        "mean_ms": 3.9578497651006708,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          27,
          205,
          61,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.045632
      },
      "copy_wall": {
        "count": 298,
        "sum_ns": 6146595249,
        "mean_ms": 20.626158553691276,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          154,
          144,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 43.151958
      },
      "copy_thread_cpu": {
        "count": 298,
        "sum_ns": 5767862258,
        "mean_ms": 19.355242476510067,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          167,
          131,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.658694
      },
      "callback_to_copied": {
        "count": 298,
        "sum_ns": 6526951841,
        "mean_ms": 21.902522956375837,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          133,
          165,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.350565
      },
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29788772219,
        "mean_ms": 99.96232288255032,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          48,
          109,
          126,
          12,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 183.998763
      },
      "selected_to_submitted": {
        "count": 298,
        "sum_ns": 18995630805,
        "mean_ms": 63.74372753355705,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          43,
          200,
          46,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 128.12733
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.12273740209639,
    "first_redis_commit_s": 24.488400229252875,
    "maximum_sampled_silence_ms": 121.506734,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 29.998858762905,
    "completed": 101,
    "accepted": 101,
    "sent": 100,
    "late": 0,
    "completed_fps": 3.3667947437017585,
    "accepted_fps": 3.3667947437017585,
    "sent_fps": 3.3334601422789687,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.912636769,
    "selected": 101,
    "infer_entries": 102,
    "infer_entry_fps": 3.4099300836530677,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 101,
        "sum_ns": 3201469392,
        "mean_ms": 31.697716752475248,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          12,
          17,
          8,
          12,
          30,
          14,
          1,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 177.287038
      },
      "accepted_interval": {
        "count": 101,
        "sum_ns": 29835028812,
        "mean_ms": 295.3963248712871,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          1,
          4,
          15,
          13,
          38,
          19,
          7,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1793.460806
      },
      "preprocess_start_to_preprocess_output": {
        "count": 101,
        "sum_ns": 4020350569,
        "mean_ms": 39.80545117821782,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          4,
          13,
          35,
          33,
          11,
          3,
          1,
          0,
          0,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1310.971029
      },
      "completed_queue_wait": {
        "count": 100,
        "sum_ns": 6739947469,
        "mean_ms": 67.39947468999999,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          7,
          17,
          15,
          10,
          5,
          5,
          7,
          7,
          8,
          6,
          11,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 342.677869
      },
      "selected_to_preprocess": {
        "count": 102,
        "sum_ns": 11884721379,
        "mean_ms": 116.51687626470589,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          30,
          18,
          17,
          20,
          7,
          5,
          1,
          0,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1500.869078
      },
      "preprocess_start_to_preprocess": {
        "count": 102,
        "sum_ns": 8594796420,
        "mean_ms": 84.26271,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          2,
          2,
          13,
          44,
          15,
          12,
          6,
          1,
          5,
          0,
          0,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1487.790438
      },
      "preprocess_output_to_infer_entry": {
        "count": 102,
        "sum_ns": 4565514018,
        "mean_ms": 44.759941352941176,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          8,
          5,
          9,
          22,
          19,
          20,
          7,
          3,
          1,
          4,
          3,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1326.522404
      },
      "copy_wall": {
        "count": 101,
        "sum_ns": 8489049281,
        "mean_ms": 84.04999288118812,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          18,
          24,
          19,
          26,
          7,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 216.572677
      },
      "copy_thread_cpu": {
        "count": 101,
        "sum_ns": 2817097317,
        "mean_ms": 27.892052643564355,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          1,
          29,
          63,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.002823
      },
      "preprocess_to_infer_output": {
        "count": 102,
        "sum_ns": 104413889750,
        "mean_ms": 1023.6655857843136,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          64,
          33,
          4,
          0
        ],
        "diagnostic_window_max_ms": 2890.105663
      },
      "preprocess_to_callback": {
        "count": 102,
        "sum_ns": 106738814484,
        "mean_ms": 1046.458965529412,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          60,
          37,
          4,
          0
        ],
        "diagnostic_window_max_ms": 2967.160894
      },
      "infer_output_to_callback": {
        "count": 102,
        "sum_ns": 2324924734,
        "mean_ms": 22.79337974509804,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          5,
          21,
          30,
          38,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 85.654548
      },
      "callback_to_copied": {
        "count": 100,
        "sum_ns": 8692350047,
        "mean_ms": 86.92350047,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          17,
          21,
          22,
          27,
          5,
          4,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 244.666713
      },
      "selected_to_submitted": {
        "count": 101,
        "sum_ns": 128304216694,
        "mean_ms": 1270.3387791485147,
        "p95_bucket_upper_ms": 5000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          27,
          64,
          10,
          0
        ],
        "diagnostic_window_max_ms": 3136.199912
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.134857448283583,
    "first_redis_commit_s": 22.24085191823542,
    "maximum_sampled_silence_ms": 563.44678,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 29.998858762905,
    "completed": 142,
    "accepted": 142,
    "sent": 142,
    "late": 0,
    "completed_fps": 4.733513402036136,
    "accepted_fps": 4.733513402036136,
    "sent_fps": 4.733513402036136,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.956382787,
    "selected": 142,
    "infer_entries": 141,
    "infer_entry_fps": 4.7068433129112295,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "accepted_interval": {
        "count": 142,
        "sum_ns": 29916622852,
        "mean_ms": 210.68044261971832,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          6,
          15,
          25,
          29,
          41,
          23,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 706.87192
      },
      "completed_queue_wait": {
        "count": 142,
        "sum_ns": 480832868,
        "mean_ms": 3.3861469577464787,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          41,
          29,
          42,
          18,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.860595
      },
      "selected_to_preprocess_start": {
        "count": 142,
        "sum_ns": 2261177646,
        "mean_ms": 15.92378623943662,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          16,
          35,
          18,
          33,
          30,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 80.676136
      },
      "preprocess_start_to_preprocess_output": {
        "count": 142,
        "sum_ns": 2285250932,
        "mean_ms": 16.09331642253521,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          3,
          34,
          65,
          39,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 49.685332
      },
      "selected_to_preprocess": {
        "count": 141,
        "sum_ns": 6985297222,
        "mean_ms": 49.541115049645384,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          14,
          70,
          30,
          20,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 145.62181
      },
      "preprocess_start_to_preprocess": {
        "count": 141,
        "sum_ns": 4733061382,
        "mean_ms": 33.56781121985816,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          7,
          26,
          83,
          14,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 142.994146
      },
      "preprocess_output_to_infer_entry": {
        "count": 141,
        "sum_ns": 2456035797,
        "mean_ms": 17.41869359574468,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          19,
          7,
          3,
          31,
          44,
          26,
          7,
          2,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 130.010278
      },
      "copy_wall": {
        "count": 142,
        "sum_ns": 9284989159,
        "mean_ms": 65.38724759859156,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          42,
          50,
          24,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 202.37879
      },
      "copy_thread_cpu": {
        "count": 142,
        "sum_ns": 2966530016,
        "mean_ms": 20.891056450704223,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          60,
          82,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.781908
      },
      "preprocess_to_infer_output": {
        "count": 141,
        "sum_ns": 109078413184,
        "mean_ms": 773.6057672624114,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          15,
          105,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1188.275807
      },
      "preprocess_to_callback": {
        "count": 140,
        "sum_ns": 110527730783,
        "mean_ms": 789.4837913071428,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          105,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1203.57035
      },
      "infer_output_to_callback": {
        "count": 141,
        "sum_ns": 2184381297,
        "mean_ms": 15.492065936170214,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          17,
          37,
          47,
          39,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.235057
      },
      "callback_to_copied": {
        "count": 141,
        "sum_ns": 9818925171,
        "mean_ms": 69.63776717021275,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          43,
          45,
          27,
          16,
          7,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 185.342403
      },
      "selected_to_submitted": {
        "count": 142,
        "sum_ns": 129462639043,
        "mean_ms": 911.7087256549295,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          87,
          52,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1459.799584
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.192372483201325,
    "first_redis_commit_s": 22.237222412135452,
    "maximum_sampled_silence_ms": 310.695961,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 29.998858762905,
    "completed": 128,
    "accepted": 128,
    "sent": 129,
    "late": 0,
    "completed_fps": 4.26682898211708,
    "accepted_fps": 4.26682898211708,
    "sent_fps": 4.30016358353987,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.877562785,
    "selected": 128,
    "infer_entries": 128,
    "infer_entry_fps": 4.284151318536004,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 124,
        "sum_ns": 1988929622,
        "mean_ms": 16.039755016129032,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          12,
          35,
          11,
          29,
          30,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 90.969383
      },
      "preprocess_start_to_preprocess_output": {
        "count": 123,
        "sum_ns": 1875600896,
        "mean_ms": 15.248787772357723,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          1,
          30,
          62,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.435269
      },
      "copy_wall": {
        "count": 128,
        "sum_ns": 7649698323,
        "mean_ms": 59.7632681484375,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          56,
          34,
          24,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 163.786841
      },
      "copy_thread_cpu": {
        "count": 128,
        "sum_ns": 2707169086,
        "mean_ms": 21.149758484375,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          47,
          81,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.63187
      },
      "selected_to_preprocess": {
        "count": 124,
        "sum_ns": 6047306382,
        "mean_ms": 48.768599854838705,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          71,
          24,
          10,
          4,
          1,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 383.239605
      },
      "preprocess_start_to_preprocess": {
        "count": 124,
        "sum_ns": 4058376760,
        "mean_ms": 32.728844838709676,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          10,
          35,
          62,
          10,
          2,
          1,
          1,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 329.593657
      },
      "preprocess_output_to_infer_entry": {
        "count": 123,
        "sum_ns": 2162698650,
        "mean_ms": 17.582915853658537,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          19,
          11,
          3,
          36,
          36,
          9,
          4,
          1,
          2,
          0,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 310.064059
      },
      "accepted_interval": {
        "count": 128,
        "sum_ns": 29591925885,
        "mean_ms": 231.1869209765625,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          4,
          6,
          25,
          22,
          40,
          24,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 851.047647
      },
      "completed_queue_wait": {
        "count": 129,
        "sum_ns": 443330065,
        "mean_ms": 3.4366671705426355,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          42,
          37,
          28,
          13,
          6,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.132613
      },
      "preprocess_to_infer_output": {
        "count": 125,
        "sum_ns": 107694931456,
        "mean_ms": 861.559451648,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          90,
          32,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1444.202712
      },
      "preprocess_to_callback": {
        "count": 125,
        "sum_ns": 109879711068,
        "mean_ms": 879.037688544,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          87,
          36,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1479.711849
      },
      "infer_output_to_callback": {
        "count": 125,
        "sum_ns": 2184779612,
        "mean_ms": 17.478236896000002,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          11,
          31,
          37,
          44,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 63.563837
      },
      "callback_to_copied": {
        "count": 125,
        "sum_ns": 7978941860,
        "mean_ms": 63.83153488,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          51,
          31,
          26,
          14,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 172.424381
      },
      "selected_to_submitted": {
        "count": 125,
        "sum_ns": 124304004384,
        "mean_ms": 994.4320350720001,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          67,
          58,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1567.145564
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.12273740209639,
    "first_redis_commit_s": 24.488400229252875,
    "maximum_sampled_silence_ms": 340.261375,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 103,
    "accepted": 103,
    "sent": 102,
    "late": 0,
    "completed_fps": 3.433463946547338,
    "accepted_fps": 3.433463946547338,
    "sent_fps": 3.4001293451245482,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.948164388,
    "selected": 103,
    "infer_entries": 103,
    "infer_entry_fps": 3.439275899035445,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 101,
        "sum_ns": 2773031776,
        "mean_ms": 27.455760158415842,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          10,
          27,
          11,
          21,
          19,
          8,
          3,
          0,
          1,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 766.62614
      },
      "preprocess_start_to_preprocess_output": {
        "count": 101,
        "sum_ns": 1819892627,
        "mean_ms": 18.018738881188117,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          3,
          22,
          44,
          30,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.025586
      },
      "selected_to_preprocess": {
        "count": 102,
        "sum_ns": 7619013685,
        "mean_ms": 74.69621259803921,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          48,
          19,
          15,
          6,
          3,
          4,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 838.13822
      },
      "preprocess_start_to_preprocess": {
        "count": 101,
        "sum_ns": 4619796469,
        "mean_ms": 45.7405590990099,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          4,
          29,
          45,
          9,
          5,
          5,
          3,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 475.346128
      },
      "preprocess_output_to_infer_entry": {
        "count": 102,
        "sum_ns": 2808126084,
        "mean_ms": 27.53064788235294,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          4,
          7,
          12,
          24,
          25,
          19,
          3,
          0,
          4,
          3,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 456.733541
      },
      "copy_wall": {
        "count": 103,
        "sum_ns": 11906834688,
        "mean_ms": 115.60033677669904,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          24,
          23,
          14,
          25,
          8,
          3,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 871.621356
      },
      "copy_thread_cpu": {
        "count": 103,
        "sum_ns": 4587422645,
        "mean_ms": 44.53808393203884,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          32,
          64,
          3,
          0,
          0,
          0,
          0,
          1,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 597.889424
      },
      "accepted_interval": {
        "count": 103,
        "sum_ns": 29790200877,
        "mean_ms": 289.2252512330097,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          1,
          12,
          21,
          31,
          22,
          9,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1033.75224
      },
      "completed_queue_wait": {
        "count": 103,
        "sum_ns": 7787949383,
        "mean_ms": 75.61115905825243,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          12,
          13,
          18,
          12,
          15,
          4,
          3,
          2,
          5,
          3,
          8,
          6,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 843.031951
      },
      "preprocess_to_infer_output": {
        "count": 102,
        "sum_ns": 100207093374,
        "mean_ms": 982.4224840588236,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          64,
          38,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1790.380901
      },
      "preprocess_to_callback": {
        "count": 102,
        "sum_ns": 102185529460,
        "mean_ms": 1001.8189162745098,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          62,
          40,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1799.337437
      },
      "infer_output_to_callback": {
        "count": 102,
        "sum_ns": 1978436086,
        "mean_ms": 19.39643221568627,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          8,
          18,
          37,
          34,
          2,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 93.664101
      },
      "callback_to_copied": {
        "count": 102,
        "sum_ns": 12270953977,
        "mean_ms": 120.30347036274509,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          24,
          19,
          16,
          22,
          11,
          4,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 872.495728
      },
      "selected_to_submitted": {
        "count": 102,
        "sum_ns": 122565472311,
        "mean_ms": 1201.6222775588235,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          32,
          69,
          1,
          0
        ],
        "diagnostic_window_max_ms": 2080.193233
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.365565648302436,
    "first_redis_commit_s": 19.670806643087417,
    "maximum_sampled_silence_ms": 421.965491,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 133,
    "accepted": 133,
    "sent": 133,
    "late": 0,
    "completed_fps": 4.433501989231028,
    "accepted_fps": 4.433501989231028,
    "sent_fps": 4.433501989231028,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.95621161,
    "selected": 133,
    "infer_entries": 133,
    "infer_entry_fps": 4.439813743190473,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 133,
        "sum_ns": 2124230540,
        "mean_ms": 15.971658195488722,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          11,
          31,
          23,
          27,
          37,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.059777
      },
      "preprocess_start_to_preprocess_output": {
        "count": 132,
        "sum_ns": 2218325219,
        "mean_ms": 16.805494083333333,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          3,
          23,
          69,
          34,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.870096
      },
      "selected_to_preprocess": {
        "count": 133,
        "sum_ns": 7128617824,
        "mean_ms": 53.5986302556391,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          71,
          27,
          14,
          10,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 247.049327
      },
      "preprocess_start_to_preprocess": {
        "count": 133,
        "sum_ns": 5004387284,
        "mean_ms": 37.62697206015038,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          7,
          32,
          68,
          12,
          7,
          5,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 233.19861
      },
      "preprocess_output_to_infer_entry": {
        "count": 132,
        "sum_ns": 2766211677,
        "mean_ms": 20.956149068181816,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          17,
          7,
          13,
          19,
          40,
          23,
          5,
          5,
          1,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 230.597131
      },
      "copy_wall": {
        "count": 133,
        "sum_ns": 9800700473,
        "mean_ms": 73.6894772406015,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          37,
          39,
          25,
          26,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 174.070517
      },
      "copy_thread_cpu": {
        "count": 133,
        "sum_ns": 2940965952,
        "mean_ms": 22.112525954887218,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          42,
          91,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.522157
      },
      "accepted_interval": {
        "count": 133,
        "sum_ns": 30014752376,
        "mean_ms": 225.67482989473686,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          6,
          9,
          21,
          25,
          44,
          21,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 778.996
      },
      "completed_queue_wait": {
        "count": 133,
        "sum_ns": 569755970,
        "mean_ms": 4.283879473684211,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          25,
          27,
          52,
          17,
          9,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.216051
      },
      "preprocess_to_infer_output": {
        "count": 132,
        "sum_ns": 105719094457,
        "mean_ms": 800.9022307348484,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          100,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1281.944294
      },
      "preprocess_to_callback": {
        "count": 132,
        "sum_ns": 107927369356,
        "mean_ms": 817.631586030303,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          7,
          101,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1299.829884
      },
      "infer_output_to_callback": {
        "count": 132,
        "sum_ns": 2208274899,
        "mean_ms": 16.729355295454546,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          17,
          25,
          54,
          30,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 63.344516
      },
      "callback_to_copied": {
        "count": 133,
        "sum_ns": 10447479690,
        "mean_ms": 78.55247887218044,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          35,
          37,
          25,
          26,
          8,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 201.703746
      },
      "selected_to_submitted": {
        "count": 133,
        "sum_ns": 126708911681,
        "mean_ms": 952.6985840676691,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          72,
          61,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1498.232051
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.281983893364668,
    "first_redis_commit_s": 19.37013702094555,
    "maximum_sampled_silence_ms": 550.242221,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 127,
    "accepted": 127,
    "sent": 127,
    "late": 0,
    "completed_fps": 4.233494380694291,
    "accepted_fps": 4.233494380694291,
    "sent_fps": 4.233494380694291,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.943334127,
    "selected": 126,
    "infer_entries": 125,
    "infer_entry_fps": 4.174551820776935,
    "pre_infer_expired": 0,
    "diagnostic_missed": 7,
    "durations": {
      "accepted_interval": {
        "count": 127,
        "sum_ns": 30217317751,
        "mean_ms": 237.93163583464565,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          2,
          9,
          17,
          20,
          46,
          29,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 849.453066
      },
      "completed_queue_wait": {
        "count": 127,
        "sum_ns": 398071815,
        "mean_ms": 3.13442374015748,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          39,
          34,
          30,
          18,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.786004
      },
      "copy_wall": {
        "count": 127,
        "sum_ns": 8505138165,
        "mean_ms": 66.9695918503937,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          48,
          35,
          22,
          16,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 199.988419
      },
      "copy_thread_cpu": {
        "count": 127,
        "sum_ns": 2808458375,
        "mean_ms": 22.113845472440943,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          38,
          89,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.821194
      },
      "selected_to_preprocess_start": {
        "count": 124,
        "sum_ns": 1558650724,
        "mean_ms": 12.569763903225805,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          13,
          35,
          16,
          35,
          23,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 60.414354
      },
      "preprocess_start_to_preprocess_output": {
        "count": 120,
        "sum_ns": 1814571659,
        "mean_ms": 15.121430491666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          8,
          30,
          54,
          24,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.262479
      },
      "selected_to_preprocess": {
        "count": 124,
        "sum_ns": 5210600152,
        "mean_ms": 42.020968967741936,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          13,
          76,
          21,
          11,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 131.482507
      },
      "preprocess_start_to_preprocess": {
        "count": 123,
        "sum_ns": 3577178166,
        "mean_ms": 29.08274931707317,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          2,
          6,
          31,
          71,
          11,
          0,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 123.700705
      },
      "preprocess_output_to_infer_entry": {
        "count": 120,
        "sum_ns": 1668103343,
        "mean_ms": 13.900861191666666,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          9,
          7,
          7,
          32,
          42,
          19,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 117.173864
      },
      "preprocess_to_infer_output": {
        "count": 124,
        "sum_ns": 107291916698,
        "mean_ms": 865.2573927258064,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          92,
          30,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1396.927413
      },
      "preprocess_to_callback": {
        "count": 124,
        "sum_ns": 109196294498,
        "mean_ms": 880.6152782096775,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          91,
          32,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1401.347909
      },
      "infer_output_to_callback": {
        "count": 125,
        "sum_ns": 1927416166,
        "mean_ms": 15.419329328,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          17,
          27,
          46,
          33,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.972461
      },
      "callback_to_copied": {
        "count": 126,
        "sum_ns": 8896130583,
        "mean_ms": 70.60421097619047,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          46,
          33,
          21,
          21,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 200.702819
      },
      "selected_to_submitted": {
        "count": 126,
        "sum_ns": 125431457826,
        "mean_ms": 995.4877605238096,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          66,
          60,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1514.099809
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.284377167001367,
    "first_redis_commit_s": 23.490568396169692,
    "maximum_sampled_silence_ms": 423.25438,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 105,
    "accepted": 105,
    "sent": 105,
    "late": 0,
    "completed_fps": 3.5001331493929175,
    "accepted_fps": 3.5001331493929175,
    "sent_fps": 3.5001331493929175,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.37859716,
    "selected": 105,
    "infer_entries": 105,
    "infer_entry_fps": 3.4563808014892548,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 104,
        "sum_ns": 2180484071,
        "mean_ms": 20.966192990384616,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          8,
          23,
          10,
          24,
          29,
          8,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.672856
      },
      "preprocess_start_to_preprocess_output": {
        "count": 104,
        "sum_ns": 3795125496,
        "mean_ms": 36.4915913076923,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          5,
          6,
          11,
          35,
          35,
          9,
          1,
          1,
          0,
          0,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1235.444751
      },
      "copy_wall": {
        "count": 105,
        "sum_ns": 9803563322,
        "mean_ms": 93.36726973333333,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          23,
          28,
          18,
          22,
          6,
          3,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1194.540443
      },
      "copy_thread_cpu": {
        "count": 105,
        "sum_ns": 3119774202,
        "mean_ms": 29.712135257142858,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          2,
          34,
          65,
          3,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 641.893399
      },
      "accepted_interval": {
        "count": 105,
        "sum_ns": 30739642018,
        "mean_ms": 292.7584954095238,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          8,
          11,
          21,
          26,
          23,
          14,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1287.497805
      },
      "selected_to_preprocess": {
        "count": 104,
        "sum_ns": 10745484225,
        "mean_ms": 103.32196370192307,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          29,
          21,
          18,
          12,
          11,
          2,
          1,
          2,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1279.93941
      },
      "preprocess_start_to_preprocess": {
        "count": 104,
        "sum_ns": 8565000154,
        "mean_ms": 82.35577071153847,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          1,
          2,
          1,
          15,
          39,
          16,
          9,
          12,
          4,
          1,
          1,
          2,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1236.133497
      },
      "preprocess_output_to_infer_entry": {
        "count": 104,
        "sum_ns": 4769874658,
        "mean_ms": 45.86417940384615,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          6,
          4,
          7,
          15,
          24,
          25,
          4,
          7,
          7,
          2,
          1,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 770.004374
      },
      "completed_queue_wait": {
        "count": 105,
        "sum_ns": 7435921877,
        "mean_ms": 70.81830359047619,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          10,
          10,
          20,
          19,
          9,
          4,
          1,
          6,
          7,
          3,
          9,
          6,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 804.381175
      },
      "preprocess_to_infer_output": {
        "count": 103,
        "sum_ns": 101010529959,
        "mean_ms": 980.6847568834951,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          62,
          40,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1956.599179
      },
      "preprocess_to_callback": {
        "count": 103,
        "sum_ns": 103255154069,
        "mean_ms": 1002.4772239708739,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          61,
          42,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1999.819504
      },
      "infer_output_to_callback": {
        "count": 103,
        "sum_ns": 2244624110,
        "mean_ms": 21.79246708737864,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          7,
          16,
          43,
          31,
          3,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 145.476815
      },
      "callback_to_copied": {
        "count": 104,
        "sum_ns": 10167134148,
        "mean_ms": 97.76090526923076,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          20,
          28,
          18,
          25,
          6,
          3,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1225.561217
      },
      "selected_to_submitted": {
        "count": 104,
        "sum_ns": 126237529806,
        "mean_ms": 1213.8224019807692,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          33,
          67,
          4,
          0
        ],
        "diagnostic_window_max_ms": 2255.761701
      }
    },
    "rejection_reasons": {},
    "first_response_s": 20.691974654328078,
    "first_redis_commit_s": 20.92268641712144,
    "maximum_sampled_silence_ms": 428.315256,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 134,
    "accepted": 134,
    "sent": 134,
    "late": 0,
    "completed_fps": 4.466836590653818,
    "accepted_fps": 4.466836590653818,
    "sent_fps": 4.466836590653818,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.905186369,
    "selected": 134,
    "infer_entries": 132,
    "infer_entry_fps": 4.413950087829329,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "copy_wall": {
        "count": 134,
        "sum_ns": 10378739101,
        "mean_ms": 77.45327687313433,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          36,
          29,
          33,
          24,
          6,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 210.138016
      },
      "copy_thread_cpu": {
        "count": 134,
        "sum_ns": 2888094005,
        "mean_ms": 21.552940335820896,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          46,
          88,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.541233
      },
      "accepted_interval": {
        "count": 134,
        "sum_ns": 29955048157,
        "mean_ms": 223.5451355,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          9,
          7,
          19,
          33,
          42,
          17,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 723.038934
      },
      "completed_queue_wait": {
        "count": 134,
        "sum_ns": 509594293,
        "mean_ms": 3.802942485074627,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          40,
          26,
          37,
          17,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.30819
      },
      "selected_to_preprocess_start": {
        "count": 134,
        "sum_ns": 2104542747,
        "mean_ms": 15.705542888059702,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          10,
          36,
          16,
          33,
          34,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.56962
      },
      "preprocess_start_to_preprocess_output": {
        "count": 133,
        "sum_ns": 2265004815,
        "mean_ms": 17.030111390977442,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          6,
          31,
          54,
          39,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.9828
      },
      "selected_to_preprocess": {
        "count": 132,
        "sum_ns": 6563567088,
        "mean_ms": 49.72399309090909,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          13,
          60,
          39,
          16,
          1,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 216.614457
      },
      "preprocess_start_to_preprocess": {
        "count": 132,
        "sum_ns": 4475214596,
        "mean_ms": 33.90314087878787,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          2,
          9,
          29,
          70,
          15,
          5,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 210.454583
      },
      "preprocess_output_to_infer_entry": {
        "count": 132,
        "sum_ns": 2221373430,
        "mean_ms": 16.82858659090909,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          22,
          7,
          15,
          18,
          36,
          27,
          4,
          1,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 202.010027
      },
      "preprocess_to_infer_output": {
        "count": 131,
        "sum_ns": 105485373387,
        "mean_ms": 805.2318579160305,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          94,
          25,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1398.685875
      },
      "preprocess_to_callback": {
        "count": 132,
        "sum_ns": 108405799889,
        "mean_ms": 821.2560597651515,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          11,
          92,
          29,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1424.038307
      },
      "infer_output_to_callback": {
        "count": 133,
        "sum_ns": 2250255727,
        "mean_ms": 16.919215992481202,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          16,
          32,
          43,
          37,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.174425
      },
      "callback_to_copied": {
        "count": 134,
        "sum_ns": 10964014038,
        "mean_ms": 81.8210002835821,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          31,
          32,
          35,
          24,
          7,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 210.765277
      },
      "selected_to_submitted": {
        "count": 134,
        "sum_ns": 127418459886,
        "mean_ms": 950.884029,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          81,
          50,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1584.347719
      }
    },
    "rejection_reasons": {},
    "first_response_s": 20.63835335895419,
    "first_redis_commit_s": 20.773864346090704,
    "maximum_sampled_silence_ms": 249.626384,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 126,
    "accepted": 126,
    "sent": 126,
    "late": 0,
    "completed_fps": 4.200159779271501,
    "accepted_fps": 4.200159779271501,
    "sent_fps": 4.200159779271501,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.010751099,
    "selected": 127,
    "infer_entries": 126,
    "infer_entry_fps": 4.198495385348703,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 127,
        "sum_ns": 1749693090,
        "mean_ms": 13.77711094488189,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          13,
          23,
          18,
          46,
          26,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 57.367649
      },
      "preprocess_start_to_preprocess_output": {
        "count": 125,
        "sum_ns": 1760495987,
        "mean_ms": 14.083967895999999,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          4,
          9,
          29,
          57,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.542943
      },
      "selected_to_preprocess": {
        "count": 126,
        "sum_ns": 5383467282,
        "mean_ms": 42.7259308095238,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          14,
          76,
          21,
          9,
          3,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 312.190844
      },
      "preprocess_start_to_preprocess": {
        "count": 126,
        "sum_ns": 3636103647,
        "mean_ms": 28.857965452380952,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          4,
          7,
          44,
          61,
          3,
          6,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 310.158113
      },
      "preprocess_output_to_infer_entry": {
        "count": 124,
        "sum_ns": 1857034272,
        "mean_ms": 14.976082838709676,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          18,
          7,
          15,
          27,
          39,
          11,
          3,
          3,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 295.801432
      },
      "copy_wall": {
        "count": 126,
        "sum_ns": 7591864668,
        "mean_ms": 60.25289419047619,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          50,
          40,
          20,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 178.174441
      },
      "copy_thread_cpu": {
        "count": 126,
        "sum_ns": 2635890404,
        "mean_ms": 20.91976511111111,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          53,
          73,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.958769
      },
      "accepted_interval": {
        "count": 126,
        "sum_ns": 30117899865,
        "mean_ms": 239.03095130952383,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          2,
          13,
          25,
          21,
          34,
          22,
          6,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1127.247463
      },
      "completed_queue_wait": {
        "count": 126,
        "sum_ns": 410509508,
        "mean_ms": 3.2580119682539683,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          42,
          31,
          27,
          18,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.523432
      },
      "preprocess_to_infer_output": {
        "count": 125,
        "sum_ns": 109031562046,
        "mean_ms": 872.252496368,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          94,
          30,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1440.527511
      },
      "preprocess_to_callback": {
        "count": 125,
        "sum_ns": 110918741305,
        "mean_ms": 887.3499304400001,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          89,
          35,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1447.608683
      },
      "infer_output_to_callback": {
        "count": 126,
        "sum_ns": 1889340770,
        "mean_ms": 14.994768015873017,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          12,
          36,
          48,
          28,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 75.242431
      },
      "callback_to_copied": {
        "count": 126,
        "sum_ns": 8114830647,
        "mean_ms": 64.40341783333334,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          48,
          38,
          20,
          16,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 208.72993
      },
      "selected_to_submitted": {
        "count": 126,
        "sum_ns": 126319952574,
        "mean_ms": 1002.5393061428572,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          64,
          62,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1614.844194
      }
    },
    "rejection_reasons": {},
    "first_response_s": 20.490891834255308,
    "first_redis_commit_s": 25.714062416926026,
    "maximum_sampled_silence_ms": 535.55454,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 105,
    "accepted": 105,
    "sent": 105,
    "late": 0,
    "completed_fps": 3.5001331493929175,
    "accepted_fps": 3.5001331493929175,
    "sent_fps": 3.5001331493929175,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.099591589,
    "selected": 105,
    "infer_entries": 104,
    "infer_entry_fps": 3.573933320745067,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "copy_wall": {
        "count": 105,
        "sum_ns": 7347278809,
        "mean_ms": 69.9740838952381,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          45,
          30,
          9,
          10,
          8,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 260.064359
      },
      "copy_thread_cpu": {
        "count": 105,
        "sum_ns": 2402683124,
        "mean_ms": 22.88269641904762,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          51,
          52,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.233842
      },
      "accepted_interval": {
        "count": 105,
        "sum_ns": 29094075902,
        "mean_ms": 277.08643716190477,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          5,
          6,
          11,
          22,
          27,
          23,
          8,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1258.578116
      },
      "completed_queue_wait": {
        "count": 105,
        "sum_ns": 7107991010,
        "mean_ms": 67.69515247619047,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          18,
          15,
          19,
          8,
          8,
          5,
          1,
          4,
          14,
          4,
          3,
          4,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 974.053519
      },
      "selected_to_preprocess_start": {
        "count": 105,
        "sum_ns": 3182169509,
        "mean_ms": 30.306376276190473,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          8,
          20,
          16,
          14,
          27,
          6,
          6,
          5,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 165.017003
      },
      "preprocess_start_to_preprocess_output": {
        "count": 104,
        "sum_ns": 2230446019,
        "mean_ms": 21.44659633653846,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          7,
          17,
          35,
          40,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.280217
      },
      "selected_to_preprocess": {
        "count": 104,
        "sum_ns": 7927064340,
        "mean_ms": 76.2217725,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          44,
          16,
          14,
          12,
          11,
          4,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 229.880424
      },
      "preprocess_start_to_preprocess": {
        "count": 104,
        "sum_ns": 4810268346,
        "mean_ms": 46.25258025,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          3,
          22,
          47,
          10,
          12,
          6,
          1,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 228.108291
      },
      "preprocess_output_to_infer_entry": {
        "count": 103,
        "sum_ns": 2559298481,
        "mean_ms": 24.847558067961163,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          8,
          5,
          10,
          21,
          24,
          20,
          5,
          6,
          2,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 211.734799
      },
      "preprocess_to_infer_output": {
        "count": 104,
        "sum_ns": 103467575003,
        "mean_ms": 994.880528875,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          61,
          41,
          1,
          0
        ],
        "diagnostic_window_max_ms": 2293.780841
      },
      "preprocess_to_callback": {
        "count": 104,
        "sum_ns": 105518057189,
        "mean_ms": 1014.5967037403846,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          60,
          42,
          1,
          0
        ],
        "diagnostic_window_max_ms": 2326.114455
      },
      "infer_output_to_callback": {
        "count": 105,
        "sum_ns": 2052032674,
        "mean_ms": 19.543168323809525,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          14,
          22,
          31,
          31,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.328781
      },
      "callback_to_copied": {
        "count": 105,
        "sum_ns": 7760131174,
        "mean_ms": 73.90601118095239,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          41,
          29,
          11,
          12,
          8,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 261.187363
      },
      "selected_to_submitted": {
        "count": 105,
        "sum_ns": 123024862002,
        "mean_ms": 1171.6653524,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          34,
          69,
          2,
          0
        ],
        "diagnostic_window_max_ms": 2442.403007
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.189985550008714,
    "first_redis_commit_s": 24.41557265818119,
    "maximum_sampled_silence_ms": 1175.200649,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 143,
    "accepted": 143,
    "sent": 142,
    "late": 0,
    "completed_fps": 4.766848003458925,
    "accepted_fps": 4.766848003458925,
    "sent_fps": 4.733513402036136,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.809849298,
    "selected": 142,
    "infer_entries": 143,
    "infer_entry_fps": 4.797072221683259,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "completed_queue_wait": {
        "count": 143,
        "sum_ns": 376913129,
        "mean_ms": 2.635756146853147,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          47,
          28,
          49,
          13,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.20032
      },
      "selected_to_preprocess_start": {
        "count": 141,
        "sum_ns": 2176613018,
        "mean_ms": 15.436971758865248,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          13,
          29,
          23,
          42,
          25,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 81.5098
      },
      "preprocess_start_to_preprocess_output": {
        "count": 141,
        "sum_ns": 2349093446,
        "mean_ms": 16.66023720567376,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          5,
          31,
          68,
          35,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.115391
      },
      "copy_wall": {
        "count": 143,
        "sum_ns": 9268715002,
        "mean_ms": 64.81618882517482,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          53,
          41,
          25,
          16,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 209.294457
      },
      "copy_thread_cpu": {
        "count": 143,
        "sum_ns": 2964785046,
        "mean_ms": 20.73276255944056,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          56,
          87,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.315787
      },
      "accepted_interval": {
        "count": 143,
        "sum_ns": 29983365667,
        "mean_ms": 209.67388578321678,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          12,
          10,
          18,
          31,
          44,
          21,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 607.320621
      },
      "selected_to_preprocess": {
        "count": 143,
        "sum_ns": 6909021603,
        "mean_ms": 48.31483638461538,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          84,
          35,
          11,
          6,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 164.250893
      },
      "preprocess_start_to_preprocess": {
        "count": 142,
        "sum_ns": 4669872800,
        "mean_ms": 32.88642816901409,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          2,
          31,
          88,
          14,
          3,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 162.597397
      },
      "preprocess_output_to_infer_entry": {
        "count": 143,
        "sum_ns": 2308185879,
        "mean_ms": 16.141159993006994,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          16,
          1,
          13,
          29,
          57,
          19,
          2,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 144.659139
      },
      "preprocess_to_infer_output": {
        "count": 142,
        "sum_ns": 109116274537,
        "mean_ms": 768.4244685704225,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          18,
          104,
          19,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1491.901685
      },
      "preprocess_to_callback": {
        "count": 141,
        "sum_ns": 110021040971,
        "mean_ms": 780.2910707163121,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          15,
          104,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1497.672741
      },
      "infer_output_to_callback": {
        "count": 141,
        "sum_ns": 2014101876,
        "mean_ms": 14.284410468085106,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          15,
          41,
          52,
          32,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 70.28059
      },
      "callback_to_copied": {
        "count": 142,
        "sum_ns": 9825363025,
        "mean_ms": 69.19269735915492,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          50,
          40,
          24,
          19,
          4,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 253.067344
      },
      "selected_to_submitted": {
        "count": 143,
        "sum_ns": 129067097860,
        "mean_ms": 902.567117902098,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          89,
          49,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1643.370424
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.000629907939583,
    "first_redis_commit_s": 24.099874312989414,
    "maximum_sampled_silence_ms": 398.386541,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 130,
    "accepted": 130,
    "sent": 130,
    "late": 0,
    "completed_fps": 4.33349818496266,
    "accepted_fps": 4.33349818496266,
    "sent_fps": 4.33349818496266,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.762791986,
    "selected": 131,
    "infer_entries": 131,
    "infer_entry_fps": 4.401468789004088,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 131,
        "sum_ns": 1958512180,
        "mean_ms": 14.950474656488549,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          8,
          27,
          18,
          38,
          38,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.54053
      },
      "preprocess_start_to_preprocess_output": {
        "count": 131,
        "sum_ns": 1949482612,
        "mean_ms": 14.881546656488549,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          1,
          33,
          74,
          21,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.293782
      },
      "copy_wall": {
        "count": 130,
        "sum_ns": 7739899521,
        "mean_ms": 59.537688623076924,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          54,
          46,
          19,
          5,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 184.692501
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2710289708,
        "mean_ms": 20.84838236923077,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          54,
          76,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.524555
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 29631735832,
        "mean_ms": 227.93642947692308,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          12,
          17,
          27,
          43,
          19,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 963.034858
      },
      "selected_to_preprocess": {
        "count": 131,
        "sum_ns": 6290968113,
        "mean_ms": 48.022657351145035,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          12,
          72,
          27,
          13,
          5,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 229.62218
      },
      "preprocess_start_to_preprocess": {
        "count": 131,
        "sum_ns": 4332455933,
        "mean_ms": 33.07218269465649,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          6,
          28,
          78,
          11,
          4,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 198.775077
      },
      "preprocess_output_to_infer_entry": {
        "count": 131,
        "sum_ns": 2382973321,
        "mean_ms": 18.190636038167938,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          10,
          4,
          10,
          26,
          54,
          17,
          6,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 190.628149
      },
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 456166563,
        "mean_ms": 3.5089735615384616,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          42,
          27,
          37,
          18,
          3,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.247262
      },
      "preprocess_to_infer_output": {
        "count": 131,
        "sum_ns": 110669539303,
        "mean_ms": 844.8056435343511,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          99,
          28,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1388.702544
      },
      "preprocess_to_callback": {
        "count": 131,
        "sum_ns": 112695406845,
        "mean_ms": 860.270281259542,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          99,
          30,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1397.063378
      },
      "infer_output_to_callback": {
        "count": 131,
        "sum_ns": 2025867542,
        "mean_ms": 15.464637725190839,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          19,
          37,
          39,
          34,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 103.40272
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 8395023768,
        "mean_ms": 64.5771059076923,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          45,
          51,
          20,
          9,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 232.891982
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 127020539420,
        "mean_ms": 977.0810724615384,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          67,
          63,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1508.875464
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.059182732366025,
    "first_redis_commit_s": 27.802249961998314,
    "maximum_sampled_silence_ms": 315.026006,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 114,
    "accepted": 114,
    "sent": 113,
    "late": 0,
    "completed_fps": 3.8001445621980245,
    "accepted_fps": 3.8001445621980245,
    "sent_fps": 3.7668099607752348,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.753644212,
    "selected": 114,
    "infer_entries": 113,
    "infer_entry_fps": 3.7978541114108553,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 114,
        "sum_ns": 2643272937,
        "mean_ms": 23.186604710526318,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          12,
          31,
          22,
          20,
          18,
          5,
          3,
          1,
          1,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 638.02302
      },
      "preprocess_start_to_preprocess_output": {
        "count": 113,
        "sum_ns": 2334852342,
        "mean_ms": 20.662410106194688,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          3,
          6,
          21,
          42,
          33,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.060318
      },
      "selected_to_preprocess": {
        "count": 113,
        "sum_ns": 8543802330,
        "mean_ms": 75.60887017699115,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          2,
          10,
          52,
          23,
          6,
          10,
          2,
          6,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 864.873466
      },
      "preprocess_start_to_preprocess": {
        "count": 113,
        "sum_ns": 5927921122,
        "mean_ms": 52.45947895575221,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          1,
          4,
          2,
          21,
          56,
          15,
          4,
          3,
          2,
          4,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 835.292922
      },
      "preprocess_output_to_infer_entry": {
        "count": 113,
        "sum_ns": 3593068780,
        "mean_ms": 31.797068849557522,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          20,
          4,
          5,
          22,
          22,
          25,
          6,
          1,
          3,
          3,
          1,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 697.232604
      },
      "copy_wall": {
        "count": 114,
        "sum_ns": 8223275007,
        "mean_ms": 72.13399128947368,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          2,
          5,
          48,
          24,
          18,
          11,
          4,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 842.066235
      },
      "copy_thread_cpu": {
        "count": 114,
        "sum_ns": 2571756691,
        "mean_ms": 22.559269219298248,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          4,
          67,
          42,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 406.74513
      },
      "accepted_interval": {
        "count": 114,
        "sum_ns": 30324385237,
        "mean_ms": 266.0033792719298,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          5,
          26,
          11,
          34,
          28,
          6,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1060.544169
      },
      "completed_queue_wait": {
        "count": 114,
        "sum_ns": 2092303965,
        "mean_ms": 18.353543552631578,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          26,
          26,
          22,
          11,
          7,
          7,
          5,
          3,
          4,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 190.398477
      },
      "preprocess_to_infer_output": {
        "count": 112,
        "sum_ns": 106503470602,
        "mean_ms": 950.9238446607143,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          69,
          41,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1736.667168
      },
      "preprocess_to_callback": {
        "count": 113,
        "sum_ns": 109274167023,
        "mean_ms": 967.0280267522124,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          69,
          43,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1749.189558
      },
      "infer_output_to_callback": {
        "count": 113,
        "sum_ns": 1525738871,
        "mean_ms": 13.502113902654868,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          21,
          28,
          41,
          16,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.23194
      },
      "callback_to_copied": {
        "count": 114,
        "sum_ns": 8433033268,
        "mean_ms": 73.97397603508772,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          2,
          5,
          46,
          25,
          18,
          11,
          4,
          1,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 842.863994
      },
      "selected_to_submitted": {
        "count": 114,
        "sum_ns": 127836764463,
        "mean_ms": 1121.3751268684211,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          52,
          62,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1867.056782
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.386767169926316,
    "first_redis_commit_s": 22.493369524832815,
    "maximum_sampled_silence_ms": 585.030255,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 142,
    "accepted": 142,
    "sent": 142,
    "late": 0,
    "completed_fps": 4.733513402036136,
    "accepted_fps": 4.733513402036136,
    "sent_fps": 4.733513402036136,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.904411825,
    "selected": 142,
    "infer_entries": 142,
    "infer_entry_fps": 4.7484632311439885,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 142,
        "sum_ns": 1485655429,
        "mean_ms": 10.462362176056338,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          40,
          34,
          18,
          25,
          21,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.523458
      },
      "preprocess_start_to_preprocess_output": {
        "count": 142,
        "sum_ns": 2517390735,
        "mean_ms": 17.72810376760563,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          2,
          28,
          65,
          45,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 62.372641
      },
      "selected_to_preprocess": {
        "count": 142,
        "sum_ns": 7479398300,
        "mean_ms": 52.67181901408451,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          11,
          83,
          24,
          11,
          6,
          2,
          5,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 273.626261
      },
      "preprocess_start_to_preprocess": {
        "count": 142,
        "sum_ns": 5993742871,
        "mean_ms": 42.20945683802817,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          4,
          32,
          81,
          7,
          7,
          4,
          4,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 251.999512
      },
      "preprocess_output_to_infer_entry": {
        "count": 142,
        "sum_ns": 3476352136,
        "mean_ms": 24.481353070422532,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          16,
          5,
          6,
          38,
          41,
          17,
          6,
          4,
          5,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 207.764007
      },
      "copy_wall": {
        "count": 142,
        "sum_ns": 8583216271,
        "mean_ms": 60.44518500704225,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          61,
          38,
          27,
          11,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 195.505237
      },
      "copy_thread_cpu": {
        "count": 142,
        "sum_ns": 2727199766,
        "mean_ms": 19.205632154929578,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          81,
          61,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.479102
      },
      "accepted_interval": {
        "count": 142,
        "sum_ns": 29798907319,
        "mean_ms": 209.85145999295773,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          9,
          11,
          22,
          34,
          34,
          23,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 638.77831
      },
      "completed_queue_wait": {
        "count": 142,
        "sum_ns": 342488247,
        "mean_ms": 2.411889063380282,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          63,
          28,
          32,
          16,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.000815
      },
      "preprocess_to_infer_output": {
        "count": 142,
        "sum_ns": 109062746373,
        "mean_ms": 768.0475096690141,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          16,
          102,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1296.881905
      },
      "preprocess_to_callback": {
        "count": 142,
        "sum_ns": 110893958234,
        "mean_ms": 780.9433678450705,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          14,
          103,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1301.709726
      },
      "infer_output_to_callback": {
        "count": 142,
        "sum_ns": 1831211861,
        "mean_ms": 12.895858176056338,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          18,
          53,
          44,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.250958
      },
      "callback_to_copied": {
        "count": 142,
        "sum_ns": 8961383192,
        "mean_ms": 63.10833233802817,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          58,
          38,
          26,
          16,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 196.658459
      },
      "selected_to_submitted": {
        "count": 142,
        "sum_ns": 127555433618,
        "mean_ms": 898.2777015352113,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          94,
          40,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1477.003953
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.173969746101648,
    "first_redis_commit_s": 22.25477125775069,
    "maximum_sampled_silence_ms": 206.908634,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 135,
    "accepted": 135,
    "sent": 135,
    "late": 0,
    "completed_fps": 4.500171192076608,
    "accepted_fps": 4.500171192076608,
    "sent_fps": 4.500171192076608,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.002514364,
    "selected": 135,
    "infer_entries": 135,
    "infer_entry_fps": 4.499622877007482,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 135,
        "sum_ns": 1560805279,
        "mean_ms": 11.561520585185185,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          27,
          33,
          15,
          35,
          23,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.870122
      },
      "preprocess_start_to_preprocess_output": {
        "count": 135,
        "sum_ns": 2049654346,
        "mean_ms": 15.182624785185185,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          4,
          35,
          63,
          33,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.360549
      },
      "selected_to_preprocess": {
        "count": 135,
        "sum_ns": 5228723366,
        "mean_ms": 38.73128419259259,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          4,
          18,
          79,
          26,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 99.092726
      },
      "preprocess_start_to_preprocess": {
        "count": 135,
        "sum_ns": 3667918087,
        "mean_ms": 27.169763607407408,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          2,
          9,
          47,
          63,
          11,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 82.436607
      },
      "preprocess_output_to_infer_entry": {
        "count": 135,
        "sum_ns": 1618263741,
        "mean_ms": 11.98713882222222,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          27,
          11,
          7,
          37,
          29,
          21,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.163118
      },
      "copy_wall": {
        "count": 135,
        "sum_ns": 7807551091,
        "mean_ms": 57.83371178518519,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          64,
          33,
          19,
          11,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 171.434487
      },
      "copy_thread_cpu": {
        "count": 135,
        "sum_ns": 2710471706,
        "mean_ms": 20.07756819259259,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          66,
          69,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.581977
      },
      "accepted_interval": {
        "count": 135,
        "sum_ns": 29927853222,
        "mean_ms": 221.68780164444445,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          3,
          6,
          29,
          31,
          33,
          26,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 707.070474
      },
      "completed_queue_wait": {
        "count": 135,
        "sum_ns": 325536714,
        "mean_ms": 2.4113830666666667,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          61,
          22,
          36,
          12,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.379465
      },
      "preprocess_to_infer_output": {
        "count": 135,
        "sum_ns": 112593280119,
        "mean_ms": 834.0242971777777,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          105,
          26,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1414.35999
      },
      "preprocess_to_callback": {
        "count": 135,
        "sum_ns": 114349723784,
        "mean_ms": 847.0349909925926,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          103,
          28,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1424.988354
      },
      "infer_output_to_callback": {
        "count": 135,
        "sum_ns": 1756443665,
        "mean_ms": 13.010693814814815,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          19,
          44,
          46,
          23,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.130374
      },
      "callback_to_copied": {
        "count": 135,
        "sum_ns": 8231556259,
        "mean_ms": 60.974490807407406,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          57,
          38,
          19,
          12,
          5,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 172.26766
      },
      "selected_to_submitted": {
        "count": 135,
        "sum_ns": 127836474458,
        "mean_ms": 946.9368478370371,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          83,
          52,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1483.494376
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.25893616490066,
    "first_redis_commit_s": 24.604763437993824,
    "maximum_sampled_silence_ms": 263.035494,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 114,
    "accepted": 114,
    "sent": 111,
    "late": 0,
    "completed_fps": 3.8001445621980245,
    "accepted_fps": 3.8001445621980245,
    "sent_fps": 3.7001407579296557,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.752721036,
    "selected": 114,
    "infer_entries": 114,
    "infer_entry_fps": 3.831582323581868,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 112,
        "sum_ns": 1504825963,
        "mean_ms": 13.435946098214286,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          20,
          24,
          16,
          26,
          23,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.505042
      },
      "copy_wall": {
        "count": 114,
        "sum_ns": 6398488798,
        "mean_ms": 56.12709471929824,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          13,
          50,
          26,
          18,
          4,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 797.165436
      },
      "copy_thread_cpu": {
        "count": 114,
        "sum_ns": 2253716957,
        "mean_ms": 19.76944699122807,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          11,
          79,
          23,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 440.408836
      },
      "accepted_interval": {
        "count": 114,
        "sum_ns": 29886232463,
        "mean_ms": 262.1599338859649,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          3,
          4,
          25,
          18,
          31,
          18,
          13,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 964.863987
      },
      "completed_queue_wait": {
        "count": 112,
        "sum_ns": 2136921429,
        "mean_ms": 19.079655616071427,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          18,
          21,
          23,
          18,
          9,
          11,
          4,
          2,
          3,
          1,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 216.730799
      },
      "preprocess_start_to_preprocess_output": {
        "count": 111,
        "sum_ns": 2213920691,
        "mean_ms": 19.94523145045045,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          9,
          27,
          50,
          17,
          4,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 536.870958
      },
      "selected_to_preprocess": {
        "count": 112,
        "sum_ns": 7163471096,
        "mean_ms": 63.959563357142855,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          6,
          18,
          45,
          17,
          12,
          7,
          3,
          1,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 736.640794
      },
      "preprocess_start_to_preprocess": {
        "count": 112,
        "sum_ns": 5658645133,
        "mean_ms": 50.523617258928574,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          1,
          3,
          10,
          30,
          43,
          6,
          10,
          5,
          0,
          1,
          1,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 734.463267
      },
      "preprocess_output_to_infer_entry": {
        "count": 111,
        "sum_ns": 3370632161,
        "mean_ms": 30.366055504504505,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          21,
          7,
          13,
          16,
          19,
          18,
          10,
          1,
          3,
          0,
          2,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 720.768317
      },
      "preprocess_to_infer_output": {
        "count": 111,
        "sum_ns": 102538780398,
        "mean_ms": 923.7727963783784,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          77,
          34,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1719.747883
      },
      "preprocess_to_callback": {
        "count": 112,
        "sum_ns": 104992963648,
        "mean_ms": 937.4371754285714,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          77,
          35,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1723.191355
      },
      "infer_output_to_callback": {
        "count": 111,
        "sum_ns": 1461728119,
        "mean_ms": 13.168721792792793,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          19,
          27,
          44,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.740583
      },
      "callback_to_copied": {
        "count": 112,
        "sum_ns": 6564649747,
        "mean_ms": 58.61294416964286,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          12,
          46,
          23,
          23,
          5,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 797.710286
      },
      "selected_to_submitted": {
        "count": 112,
        "sum_ns": 119188233510,
        "mean_ms": 1064.1806563392856,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          56,
          56,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1859.565751
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.684864268172532,
    "first_redis_commit_s": 23.733478801324964,
    "maximum_sampled_silence_ms": 273.188484,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 138,
    "accepted": 138,
    "sent": 138,
    "late": 0,
    "completed_fps": 4.600174996344977,
    "accepted_fps": 4.600174996344977,
    "sent_fps": 4.600174996344977,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.63047596,
    "selected": 138,
    "infer_entries": 138,
    "infer_entry_fps": 4.6573669685999874,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "accepted_interval": {
        "count": 138,
        "sum_ns": 29722820479,
        "mean_ms": 215.3827570942029,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          5,
          17,
          22,
          27,
          36,
          22,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 765.635195
      },
      "completed_queue_wait": {
        "count": 138,
        "sum_ns": 427961468,
        "mean_ms": 3.101170057971015,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          53,
          17,
          39,
          20,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.953623
      },
      "selected_to_preprocess_start": {
        "count": 136,
        "sum_ns": 1323584084,
        "mean_ms": 9.732235911764706,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          23,
          34,
          26,
          36,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.085856
      },
      "copy_wall": {
        "count": 138,
        "sum_ns": 8397927523,
        "mean_ms": 60.854547268115944,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          54,
          42,
          23,
          12,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.933203
      },
      "copy_thread_cpu": {
        "count": 138,
        "sum_ns": 2739688292,
        "mean_ms": 19.852813710144925,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          67,
          71,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.907066
      },
      "preprocess_start_to_preprocess_output": {
        "count": 136,
        "sum_ns": 1974464167,
        "mean_ms": 14.518118875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          5,
          37,
          58,
          32,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.755139
      },
      "selected_to_preprocess": {
        "count": 136,
        "sum_ns": 5261407315,
        "mean_ms": 38.686818492647056,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          16,
          85,
          25,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 143.77101
      },
      "preprocess_start_to_preprocess": {
        "count": 136,
        "sum_ns": 3937823231,
        "mean_ms": 28.954582580882352,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          12,
          33,
          75,
          10,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.947048
      },
      "preprocess_output_to_infer_entry": {
        "count": 136,
        "sum_ns": 1963359064,
        "mean_ms": 14.436463705882353,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          20,
          14,
          12,
          22,
          40,
          20,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 109.4331
      },
      "preprocess_to_infer_output": {
        "count": 137,
        "sum_ns": 107321490387,
        "mean_ms": 783.3685429708029,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          10,
          103,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1332.801215
      },
      "preprocess_to_callback": {
        "count": 137,
        "sum_ns": 108956011890,
        "mean_ms": 795.2993568613139,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          104,
          25,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1339.741418
      },
      "infer_output_to_callback": {
        "count": 137,
        "sum_ns": 1634521503,
        "mean_ms": 11.930813890510949,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          28,
          45,
          39,
          23,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.455748
      },
      "callback_to_copied": {
        "count": 136,
        "sum_ns": 8674159239,
        "mean_ms": 63.780582639705884,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          49,
          41,
          25,
          14,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 192.67126
      },
      "selected_to_submitted": {
        "count": 136,
        "sum_ns": 122397054593,
        "mean_ms": 899.9783425955882,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          98,
          38,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1462.377085
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.480934132356197,
    "first_redis_commit_s": 23.500236039981246,
    "maximum_sampled_silence_ms": 540.139453,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 127,
    "accepted": 127,
    "sent": 126,
    "late": 0,
    "completed_fps": 4.233494380694291,
    "accepted_fps": 4.233494380694291,
    "sent_fps": 4.200159779271501,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.592874215,
    "selected": 127,
    "infer_entries": 127,
    "infer_entry_fps": 4.291573676734192,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "accepted_interval": {
        "count": 127,
        "sum_ns": 29705320097,
        "mean_ms": 233.90015824409448,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          6,
          12,
          18,
          23,
          36,
          24,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 784.566392
      },
      "completed_queue_wait": {
        "count": 127,
        "sum_ns": 366617171,
        "mean_ms": 2.8867493779527558,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          50,
          20,
          39,
          11,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.965375
      },
      "selected_to_preprocess_start": {
        "count": 126,
        "sum_ns": 1165860636,
        "mean_ms": 9.25286219047619,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          35,
          32,
          19,
          22,
          17,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.061368
      },
      "copy_wall": {
        "count": 127,
        "sum_ns": 8127523913,
        "mean_ms": 63.99625128346457,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          45,
          38,
          25,
          13,
          1,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 211.962523
      },
      "copy_thread_cpu": {
        "count": 127,
        "sum_ns": 2541172797,
        "mean_ms": 20.009234622047245,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          66,
          61,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.245748
      },
      "preprocess_start_to_preprocess_output": {
        "count": 126,
        "sum_ns": 1920524402,
        "mean_ms": 15.242257158730158,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          9,
          23,
          65,
          26,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.952289
      },
      "selected_to_preprocess": {
        "count": 127,
        "sum_ns": 4629659955,
        "mean_ms": 36.454015393700786,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          5,
          21,
          75,
          16,
          6,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 113.413533
      },
      "preprocess_start_to_preprocess": {
        "count": 126,
        "sum_ns": 3428196117,
        "mean_ms": 27.207905690476192,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          4,
          11,
          38,
          60,
          8,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 111.597244
      },
      "preprocess_output_to_infer_entry": {
        "count": 126,
        "sum_ns": 1524494940,
        "mean_ms": 12.09916619047619,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          29,
          8,
          14,
          24,
          30,
          17,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 104.434833
      },
      "preprocess_to_infer_output": {
        "count": 127,
        "sum_ns": 109727367285,
        "mean_ms": 863.995017992126,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          92,
          33,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1392.134524
      },
      "preprocess_to_callback": {
        "count": 127,
        "sum_ns": 111306209617,
        "mean_ms": 876.4268473779529,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          93,
          34,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1437.105963
      },
      "infer_output_to_callback": {
        "count": 127,
        "sum_ns": 1578842332,
        "mean_ms": 12.431829385826772,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          23,
          36,
          49,
          19,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.971439
      },
      "callback_to_copied": {
        "count": 127,
        "sum_ns": 8478717725,
        "mean_ms": 66.76155688976378,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          43,
          38,
          25,
          16,
          1,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 215.561068
      },
      "selected_to_submitted": {
        "count": 127,
        "sum_ns": 125086076417,
        "mean_ms": 984.9297355669291,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          70,
          57,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1575.656298
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.386513303034008,
    "first_redis_commit_s": 26.299603417050093,
    "maximum_sampled_silence_ms": 231.128824,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 109,
    "accepted": 109,
    "sent": 110,
    "late": 0,
    "completed_fps": 3.633471555084076,
    "accepted_fps": 3.633471555084076,
    "sent_fps": 3.666806156506866,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.3784057,
    "selected": 110,
    "infer_entries": 110,
    "infer_entry_fps": 3.7442467478757706,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 109,
        "sum_ns": 1566320162,
        "mean_ms": 14.369909743119265,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          20,
          27,
          17,
          19,
          21,
          3,
          0,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 149.762403
      },
      "copy_wall": {
        "count": 109,
        "sum_ns": 7052253042,
        "mean_ms": 64.69956919266055,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          3,
          9,
          41,
          27,
          17,
          8,
          2,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 601.850776
      },
      "copy_thread_cpu": {
        "count": 109,
        "sum_ns": 2467152409,
        "mean_ms": 22.634425770642203,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          11,
          73,
          23,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 363.566889
      },
      "accepted_interval": {
        "count": 109,
        "sum_ns": 29109515130,
        "mean_ms": 267.0597718348624,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          4,
          7,
          20,
          16,
          25,
          21,
          14,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 866.175806
      },
      "preprocess_start_to_preprocess_output": {
        "count": 108,
        "sum_ns": 2035154067,
        "mean_ms": 18.84401913888889,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          2,
          2,
          7,
          22,
          37,
          32,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.726986
      },
      "completed_queue_wait": {
        "count": 109,
        "sum_ns": 3469231435,
        "mean_ms": 31.82781133027523,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          22,
          20,
          18,
          12,
          7,
          10,
          3,
          4,
          8,
          4,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 630.366996
      },
      "selected_to_preprocess": {
        "count": 109,
        "sum_ns": 6747071468,
        "mean_ms": 61.89973823853211,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          6,
          11,
          44,
          18,
          11,
          10,
          6,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 370.471762
      },
      "preprocess_start_to_preprocess": {
        "count": 109,
        "sum_ns": 5180657298,
        "mean_ms": 47.52896603669725,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          1,
          4,
          11,
          20,
          42,
          11,
          6,
          7,
          5,
          1,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 342.640949
      },
      "preprocess_output_to_infer_entry": {
        "count": 109,
        "sum_ns": 3143646018,
        "mean_ms": 28.8407891559633,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          11,
          8,
          10,
          22,
          25,
          14,
          8,
          2,
          6,
          2,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 330.276147
      },
      "preprocess_to_infer_output": {
        "count": 109,
        "sum_ns": 103038152826,
        "mean_ms": 945.3041543669724,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          68,
          40,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1642.208532
      },
      "preprocess_to_callback": {
        "count": 109,
        "sum_ns": 104493851636,
        "mean_ms": 958.6591893211009,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          67,
          42,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1665.403603
      },
      "infer_output_to_callback": {
        "count": 109,
        "sum_ns": 1455698810,
        "mean_ms": 13.35503495412844,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          17,
          35,
          31,
          21,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.474249
      },
      "callback_to_copied": {
        "count": 108,
        "sum_ns": 7226964887,
        "mean_ms": 66.9163415462963,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          9,
          39,
          25,
          22,
          7,
          3,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 604.162182
      },
      "selected_to_submitted": {
        "count": 108,
        "sum_ns": 118289097600,
        "mean_ms": 1095.2694222222224,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          51,
          57,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1732.050744
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.82604376785457,
    "first_redis_commit_s": 22.003038201015443,
    "maximum_sampled_silence_ms": 281.832538,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 139,
    "accepted": 139,
    "sent": 139,
    "late": 0,
    "completed_fps": 4.633509597767767,
    "accepted_fps": 4.633509597767767,
    "sent_fps": 4.633509597767767,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.644642321,
    "selected": 139,
    "infer_entries": 139,
    "infer_entry_fps": 4.688874249008349,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "copy_wall": {
        "count": 139,
        "sum_ns": 8966796044,
        "mean_ms": 64.50932405755395,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          57,
          35,
          23,
          16,
          3,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 311.782736
      },
      "copy_thread_cpu": {
        "count": 139,
        "sum_ns": 2850014877,
        "mean_ms": 20.503704151079138,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          75,
          63,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 170.151132
      },
      "accepted_interval": {
        "count": 139,
        "sum_ns": 29569099617,
        "mean_ms": 212.7273353741007,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          6,
          8,
          36,
          17,
          44,
          22,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 610.352427
      },
      "selected_to_preprocess_start": {
        "count": 138,
        "sum_ns": 1299825728,
        "mean_ms": 9.419027014492753,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          37,
          33,
          18,
          28,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.167771
      },
      "completed_queue_wait": {
        "count": 139,
        "sum_ns": 454889034,
        "mean_ms": 3.272582978417266,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          42,
          25,
          49,
          16,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 19.672473
      },
      "preprocess_start_to_preprocess_output": {
        "count": 137,
        "sum_ns": 2048457967,
        "mean_ms": 14.95224793430657,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          7,
          36,
          56,
          35,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.822696
      },
      "selected_to_preprocess": {
        "count": 139,
        "sum_ns": 5243467428,
        "mean_ms": 37.722787251798565,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          3,
          24,
          82,
          25,
          2,
          2,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.536885
      },
      "preprocess_start_to_preprocess": {
        "count": 138,
        "sum_ns": 3849397421,
        "mean_ms": 27.894184210144925,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          5,
          6,
          48,
          66,
          9,
          3,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 209.591551
      },
      "preprocess_output_to_infer_entry": {
        "count": 137,
        "sum_ns": 1787999173,
        "mean_ms": 13.051088854014598,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          22,
          7,
          23,
          29,
          33,
          18,
          1,
          3,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 199.269506
      },
      "preprocess_to_infer_output": {
        "count": 139,
        "sum_ns": 106945953490,
        "mean_ms": 769.3953488489208,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          106,
          20,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1517.548117
      },
      "preprocess_to_callback": {
        "count": 139,
        "sum_ns": 108438787853,
        "mean_ms": 780.135164410072,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          106,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1524.931489
      },
      "infer_output_to_callback": {
        "count": 139,
        "sum_ns": 1492834363,
        "mean_ms": 10.73981556115108,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          27,
          52,
          42,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.132289
      },
      "callback_to_copied": {
        "count": 139,
        "sum_ns": 9380182866,
        "mean_ms": 67.48332997122303,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          49,
          38,
          27,
          18,
          3,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 312.484633
      },
      "selected_to_submitted": {
        "count": 139,
        "sum_ns": 123213349280,
        "mean_ms": 886.42697323741,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          93,
          42,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1623.02791
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.61916891578585,
    "first_redis_commit_s": 21.6353015829809,
    "maximum_sampled_silence_ms": 233.35709,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 125,
    "accepted": 125,
    "sent": 125,
    "late": 0,
    "completed_fps": 4.166825177848711,
    "accepted_fps": 4.166825177848711,
    "sent_fps": 4.166825177848711,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.328454157,
    "selected": 125,
    "infer_entries": 125,
    "infer_entry_fps": 4.262072570577863,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 123,
        "sum_ns": 1143545038,
        "mean_ms": 9.297114130081301,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          30,
          31,
          23,
          18,
          21,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.994575
      },
      "preprocess_start_to_preprocess_output": {
        "count": 122,
        "sum_ns": 1984206704,
        "mean_ms": 16.263989377049178,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          3,
          36,
          52,
          27,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 62.809908
      },
      "copy_wall": {
        "count": 125,
        "sum_ns": 7897450975,
        "mean_ms": 63.1796078,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          55,
          31,
          20,
          16,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.333021
      },
      "copy_thread_cpu": {
        "count": 125,
        "sum_ns": 2453349764,
        "mean_ms": 19.626798112,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          60,
          65,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.010653
      },
      "accepted_interval": {
        "count": 125,
        "sum_ns": 29279973715,
        "mean_ms": 234.23978972,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          9,
          9,
          17,
          18,
          36,
          27,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 698.557235
      },
      "completed_queue_wait": {
        "count": 125,
        "sum_ns": 379252452,
        "mean_ms": 3.034019616,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          42,
          32,
          28,
          13,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.701015
      },
      "selected_to_preprocess": {
        "count": 123,
        "sum_ns": 4921596130,
        "mean_ms": 40.01297666666667,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          1,
          23,
          65,
          22,
          8,
          1,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 228.786549
      },
      "preprocess_start_to_preprocess": {
        "count": 123,
        "sum_ns": 3778051092,
        "mean_ms": 30.715862536585366,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          1,
          10,
          35,
          60,
          12,
          1,
          1,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 217.058146
      },
      "preprocess_output_to_infer_entry": {
        "count": 122,
        "sum_ns": 1782643967,
        "mean_ms": 14.611835795081968,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          15,
          9,
          18,
          33,
          26,
          17,
          0,
          2,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 209.169054
      },
      "preprocess_to_infer_output": {
        "count": 123,
        "sum_ns": 105819992126,
        "mean_ms": 860.3251392357723,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          90,
          29,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1411.907301
      },
      "preprocess_to_callback": {
        "count": 123,
        "sum_ns": 107220034723,
        "mean_ms": 871.7075993739837,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          86,
          34,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1421.033762
      },
      "infer_output_to_callback": {
        "count": 123,
        "sum_ns": 1400042597,
        "mean_ms": 11.382460138211382,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          20,
          43,
          47,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.379002
      },
      "callback_to_copied": {
        "count": 123,
        "sum_ns": 8027122612,
        "mean_ms": 65.26115944715447,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          50,
          32,
          22,
          16,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 222.230412
      },
      "selected_to_submitted": {
        "count": 123,
        "sum_ns": 120218990471,
        "mean_ms": 977.3901664308943,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          59,
          64,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1574.128801
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.79168018978089,
    "first_redis_commit_s": 24.997748056892306,
    "maximum_sampled_silence_ms": 315.823428,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 111,
    "accepted": 111,
    "sent": 109,
    "late": 0,
    "completed_fps": 3.7001407579296557,
    "accepted_fps": 3.7001407579296557,
    "sent_fps": 3.633471555084076,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 28.980851765,
    "selected": 111,
    "infer_entries": 109,
    "infer_entry_fps": 3.7611040863760477,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 111,
        "sum_ns": 1640297927,
        "mean_ms": 14.777458801801803,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          20,
          28,
          15,
          20,
          22,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.962165
      },
      "preprocess_start_to_preprocess_output": {
        "count": 110,
        "sum_ns": 2088182499,
        "mean_ms": 18.983477263636363,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          4,
          26,
          44,
          32,
          2,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 93.463194
      },
      "selected_to_preprocess": {
        "count": 109,
        "sum_ns": 6807010788,
        "mean_ms": 62.44964025688073,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          13,
          45,
          22,
          10,
          10,
          6,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 241.364095
      },
      "preprocess_start_to_preprocess": {
        "count": 109,
        "sum_ns": 5170803356,
        "mean_ms": 47.43856289908257,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          8,
          22,
          44,
          19,
          3,
          7,
          4,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 207.303069
      },
      "preprocess_output_to_infer_entry": {
        "count": 109,
        "sum_ns": 3099621302,
        "mean_ms": 28.436892678899085,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          9,
          4,
          14,
          18,
          28,
          18,
          5,
          4,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 182.880836
      },
      "copy_wall": {
        "count": 111,
        "sum_ns": 6380289160,
        "mean_ms": 57.48008252252252,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          21,
          41,
          24,
          12,
          8,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 525.323045
      },
      "copy_thread_cpu": {
        "count": 111,
        "sum_ns": 2187524994,
        "mean_ms": 19.707432378378382,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          24,
          76,
          9,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 358.383235
      },
      "accepted_interval": {
        "count": 111,
        "sum_ns": 29116340520,
        "mean_ms": 262.30937405405405,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          2,
          7,
          18,
          16,
          31,
          22,
          12,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 848.76936
      },
      "completed_queue_wait": {
        "count": 111,
        "sum_ns": 2405959052,
        "mean_ms": 21.675306774774775,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          19,
          18,
          29,
          11,
          10,
          8,
          6,
          1,
          4,
          3,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 219.176574
      },
      "preprocess_to_infer_output": {
        "count": 110,
        "sum_ns": 101078728933,
        "mean_ms": 918.8975357545454,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          72,
          37,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1799.632927
      },
      "preprocess_to_callback": {
        "count": 110,
        "sum_ns": 102597107303,
        "mean_ms": 932.7009754818182,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          71,
          38,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1806.540737
      },
      "infer_output_to_callback": {
        "count": 111,
        "sum_ns": 1528723940,
        "mean_ms": 13.772287747747747,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          24,
          32,
          33,
          18,
          2,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 134.750267
      },
      "callback_to_copied": {
        "count": 111,
        "sum_ns": 6673110842,
        "mean_ms": 60.11811569369369,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          17,
          39,
          29,
          12,
          9,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 526.582293
      },
      "selected_to_submitted": {
        "count": 111,
        "sum_ns": 117358564650,
        "mean_ms": 1057.2843662162163,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          55,
          55,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1925.067761
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.894332223106176,
    "first_redis_commit_s": 24.158745741005987,
    "maximum_sampled_silence_ms": 379.836786,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 130,
    "accepted": 130,
    "sent": 130,
    "late": 0,
    "completed_fps": 4.33349818496266,
    "accepted_fps": 4.33349818496266,
    "sent_fps": 4.33349818496266,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 28.997536357,
    "selected": 129,
    "infer_entries": 130,
    "infer_entry_fps": 4.483139477765255,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 128,
        "sum_ns": 1938178456,
        "mean_ms": 15.1420191875,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          10,
          30,
          23,
          29,
          35,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 124.339387
      },
      "copy_wall": {
        "count": 130,
        "sum_ns": 10470109060,
        "mean_ms": 80.53930046153846,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          34,
          31,
          28,
          26,
          8,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 227.582129
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2816462241,
        "mean_ms": 21.665094161538462,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          47,
          83,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.243041
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 28880157164,
        "mean_ms": 222.1550551076923,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          2,
          8,
          10,
          17,
          32,
          38,
          16,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 770.115691
      },
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 542133413,
        "mean_ms": 4.170257023076923,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          36,
          29,
          35,
          19,
          7,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.556094
      },
      "preprocess_start_to_preprocess_output": {
        "count": 127,
        "sum_ns": 2272011703,
        "mean_ms": 17.88985592913386,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          6,
          28,
          51,
          38,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.793633
      },
      "selected_to_preprocess": {
        "count": 129,
        "sum_ns": 6569882671,
        "mean_ms": 50.929323031007755,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          66,
          37,
          13,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.823061
      },
      "preprocess_start_to_preprocess": {
        "count": 129,
        "sum_ns": 4609048139,
        "mean_ms": 35.72905534108527,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          10,
          22,
          67,
          23,
          3,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 122.353013
      },
      "preprocess_output_to_infer_entry": {
        "count": 128,
        "sum_ns": 2273845481,
        "mean_ms": 17.7644178203125,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          12,
          5,
          15,
          21,
          40,
          27,
          3,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 116.004599
      },
      "preprocess_to_infer_output": {
        "count": 128,
        "sum_ns": 102139923330,
        "mean_ms": 797.968151015625,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          94,
          26,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1373.117341
      },
      "preprocess_to_callback": {
        "count": 127,
        "sum_ns": 102940052404,
        "mean_ms": 810.5515937322834,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          92,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1386.25195
      },
      "infer_output_to_callback": {
        "count": 127,
        "sum_ns": 1583186759,
        "mean_ms": 12.466037472440945,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          19,
          53,
          32,
          22,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.356063
      },
      "callback_to_copied": {
        "count": 128,
        "sum_ns": 10684189516,
        "mean_ms": 83.47023059375,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          27,
          31,
          30,
          26,
          11,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 228.507604
      },
      "selected_to_submitted": {
        "count": 129,
        "sum_ns": 122159728446,
        "mean_ms": 946.9746391162791,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          80,
          49,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1505.490752
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.1963035101071,
    "first_redis_commit_s": 24.301409311126918,
    "maximum_sampled_silence_ms": 438.276818,
    "redis_commit_p95_bucket_upper_ms": 1000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.998858762905,
    "completed": 126,
    "accepted": 126,
    "sent": 125,
    "late": 0,
    "completed_fps": 4.200159779271501,
    "accepted_fps": 4.200159779271501,
    "sent_fps": 4.166825177848711,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.16417285,
    "selected": 125,
    "infer_entries": 124,
    "infer_entry_fps": 4.251792109372305,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 125,
        "sum_ns": 1356137592,
        "mean_ms": 10.849100735999999,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          21,
          31,
          20,
          33,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 43.174538
      },
      "preprocess_start_to_preprocess_output": {
        "count": 124,
        "sum_ns": 1921349451,
        "mean_ms": 15.494753637096775,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          7,
          29,
          53,
          32,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.883201
      },
      "selected_to_preprocess": {
        "count": 124,
        "sum_ns": 5186472354,
        "mean_ms": 41.8263899516129,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          4,
          16,
          70,
          21,
          8,
          3,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 195.913639
      },
      "preprocess_start_to_preprocess": {
        "count": 124,
        "sum_ns": 3847685620,
        "mean_ms": 31.029722741935483,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          10,
          34,
          64,
          8,
          3,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 159.290895
      },
      "preprocess_output_to_infer_entry": {
        "count": 123,
        "sum_ns": 1881535331,
        "mean_ms": 15.297035211382113,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          21,
          7,
          17,
          25,
          31,
          13,
          4,
          3,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 132.527744
      },
      "accepted_interval": {
        "count": 126,
        "sum_ns": 29395266326,
        "mean_ms": 233.29576449206348,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          7,
          7,
          22,
          26,
          40,
          13,
          10,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 880.811325
      },
      "completed_queue_wait": {
        "count": 126,
        "sum_ns": 407466613,
        "mean_ms": 3.233862007936508,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          38,
          27,
          36,
          18,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.646203
      },
      "copy_wall": {
        "count": 126,
        "sum_ns": 7662816658,
        "mean_ms": 60.81600522222222,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          57,
          31,
          17,
          15,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 180.653181
      },
      "copy_thread_cpu": {
        "count": 126,
        "sum_ns": 2566805872,
        "mean_ms": 20.371475174603177,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          55,
          71,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.401662
      },
      "preprocess_to_infer_output": {
        "count": 123,
        "sum_ns": 104906560762,
        "mean_ms": 852.8988679837398,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          93,
          28,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1400.253837
      },
      "preprocess_to_callback": {
        "count": 123,
        "sum_ns": 106508538194,
        "mean_ms": 865.9230747479675,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          89,
          32,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1405.081136
      },
      "infer_output_to_callback": {
        "count": 124,
        "sum_ns": 1605769407,
        "mean_ms": 12.949753282258065,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          22,
          31,
          49,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.976033
      },
      "callback_to_copied": {
        "count": 124,
        "sum_ns": 7865044555,
        "mean_ms": 63.42777866935484,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          52,
          32,
          18,
          16,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 200.301253
      },
      "selected_to_submitted": {
        "count": 124,
        "sum_ns": 120802499127,
        "mean_ms": 974.2137026370967,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          69,
          55,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1543.352028
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.59383345907554,
    "first_redis_commit_s": 26.465257135219872,
    "maximum_sampled_silence_ms": 238.297492,
    "redis_commit_p95_bucket_upper_ms": 1000
  }
]