[
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.00016605388373,
    "completed": 300,
    "accepted": 300,
    "sent": 300,
    "late": 0,
    "completed_fps": 9.999944649011798,
    "accepted_fps": 9.999944649011798,
    "sent_fps": 9.999944649011798,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.963197198,
    "selected": 297,
    "infer_entries": 299,
    "infer_entry_fps": 9.978908392992114,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "accepted_interval": {
        "count": 300,
        "sum_ns": 29945943071,
        "mean_ms": 99.81981023666667,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          11,
          41,
          34,
          103,
          96,
          4,
          0,
          4,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 658.570787
      },
      "completed_queue_wait": {
        "count": 299,
        "sum_ns": 16259010745,
        "mean_ms": 54.37796235785953,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          124,
          48,
          28,
          1,
          5,
          8,
          11,
          12,
          17,
          17,
          13,
          15,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 469.797677
      },
      "selected_to_preprocess_start": {
        "count": 299,
        "sum_ns": 1936466977,
        "mean_ms": 6.476478183946488,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          85,
          125,
          54,
          29,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 526.92417
      },
      "preprocess_start_to_preprocess_output": {
        "count": 299,
        "sum_ns": 1647984863,
        "mean_ms": 5.511655060200669,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          52,
          91,
          79,
          57,
          15,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 503.877431
      },
      "selected_to_preprocess": {
        "count": 299,
        "sum_ns": 4455665788,
        "mean_ms": 14.901892267558528,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          90,
          98,
          59,
          50,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 548.57264
      },
      "preprocess_start_to_preprocess": {
        "count": 299,
        "sum_ns": 2519198811,
        "mean_ms": 8.42541408361204,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          79,
          92,
          61,
          37,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 504.574919
      },
      "preprocess_output_to_infer_entry": {
        "count": 299,
        "sum_ns": 871213948,
        "mean_ms": 2.9137590234113713,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          173,
          27,
          46,
          31,
          16,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.091553
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 14605006177,
        "mean_ms": 48.84617450501672,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          209,
          59,
          12,
          14,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 225.280343
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 15815453699,
        "mean_ms": 52.89449397658863,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          194,
          69,
          16,
          14,
          5,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 234.382574
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 1210447522,
        "mean_ms": 4.048319471571906,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          73,
          160,
          45,
          21,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.139177
      },
      "copy_wall": {
        "count": 299,
        "sum_ns": 4730383594,
        "mean_ms": 15.82068091638796,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          18,
          116,
          109,
          52,
          2,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 389.556551
      },
      "copy_thread_cpu": {
        "count": 299,
        "sum_ns": 4037835655,
        "mean_ms": 13.504467073578596,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          20,
          132,
          125,
          20,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 383.365183
      },
      "callback_to_copied": {
        "count": 299,
        "sum_ns": 4994766946,
        "mean_ms": 16.70490617391304,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          11,
          106,
          121,
          57,
          2,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 390.219222
      },
      "selected_to_submitted": {
        "count": 299,
        "sum_ns": 25505277393,
        "mean_ms": 85.30193108026756,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          26,
          162,
          55,
          36,
          13,
          3,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 633.999828
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.32267033495009,
    "first_redis_commit_s": 22.42273849574849,
    "maximum_sampled_silence_ms": 352.780726,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.00016605388373,
    "completed": 296,
    "accepted": 296,
    "sent": 299,
    "late": 0,
    "completed_fps": 9.866612053691641,
    "accepted_fps": 9.866612053691641,
    "sent_fps": 9.966611500181758,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.663388605,
    "selected": 296,
    "infer_entries": 296,
    "infer_entry_fps": 9.97863069326162,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "completed_queue_wait": {
        "count": 297,
        "sum_ns": 330138947,
        "mean_ms": 1.111578946127946,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          204,
          57,
          31,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 9.345422
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 851821039,
        "mean_ms": 2.8777737804054055,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          99,
          171,
          22,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.955961
      },
      "preprocess_start_to_preprocess_output": {
        "count": 296,
        "sum_ns": 530884189,
        "mean_ms": 1.7935276655405405,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          78,
          147,
          60,
          8,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 16.998503
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 1581690820,
        "mean_ms": 5.343550067567567,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          172,
          105,
          15,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.325075
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 729869781,
        "mean_ms": 2.4657762871621625,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          4,
          161,
          111,
          16,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.716259
      },
      "preprocess_output_to_infer_entry": {
        "count": 296,
        "sum_ns": 198985592,
        "mean_ms": 0.6722486216216217,
        "p95_bucket_upper_ms": 2,
        "histogram": [
          272,
          14,
          7,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 6.661065
      },
      "preprocess_to_infer_output": {
        "count": 296,
        "sum_ns": 8923631262,
        "mean_ms": 30.147402912162164,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          24,
          268,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 65.386796
      },
      "preprocess_to_callback": {
        "count": 296,
        "sum_ns": 9978893999,
        "mean_ms": 33.71247972635135,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          8,
          274,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 69.147149
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 1055262737,
        "mean_ms": 3.5650768141891893,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          53,
          202,
          38,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 15.468105
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 5825510830,
        "mean_ms": 19.680779831081082,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          172,
          124,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.099396
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5462142400,
        "mean_ms": 18.453183783783782,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          204,
          92,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.054612
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 6152034029,
        "mean_ms": 20.78389874662162,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          142,
          154,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 43.540337
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29653356248,
        "mean_ms": 100.1802575945946,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          40,
          95,
          151,
          5,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 169.576615
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 17885125016,
        "mean_ms": 60.42271964864865,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          43,
          231,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 99.562621
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.058295923750848,
    "first_redis_commit_s": 22.104469902813435,
    "maximum_sampled_silence_ms": 137.52238,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 1,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.00016605388373,
    "completed": 301,
    "accepted": 301,
    "sent": 299,
    "late": 0,
    "completed_fps": 10.033277797841837,
    "accepted_fps": 10.033277797841837,
    "sent_fps": 9.966611500181758,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.905903858,
    "selected": 301,
    "infer_entries": 299,
    "infer_entry_fps": 9.9980258553535,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 299,
        "sum_ns": 879699843,
        "mean_ms": 2.942139943143813,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          98,
          168,
          29,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.667175
      },
      "preprocess_start_to_preprocess_output": {
        "count": 299,
        "sum_ns": 527204021,
        "mean_ms": 1.7632241505016721,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          73,
          156,
          58,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.63894
      },
      "selected_to_preprocess": {
        "count": 299,
        "sum_ns": 1606810235,
        "mean_ms": 5.373947274247492,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          0,
          179,
          102,
          14,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.107427
      },
      "preprocess_start_to_preprocess": {
        "count": 299,
        "sum_ns": 727110392,
        "mean_ms": 2.4318073311036787,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          4,
          171,
          100,
          20,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.094645
      },
      "preprocess_output_to_infer_entry": {
        "count": 299,
        "sum_ns": 199906371,
        "mean_ms": 0.6685831806020067,
        "p95_bucket_upper_ms": 2,
        "histogram": [
          277,
          11,
          9,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.188446
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 8867617024,
        "mean_ms": 29.65758202006689,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          28,
          269,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 58.489858
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 9904323798,
        "mean_ms": 33.124828755852846,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          11,
          280,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.257513
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 1036706774,
        "mean_ms": 3.4672467357859533,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          53,
          203,
          42,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 12.489619
      },
      "copy_wall": {
        "count": 298,
        "sum_ns": 5806636407,
        "mean_ms": 19.4853570704698,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          168,
          130,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 37.959718
      },
      "copy_thread_cpu": {
        "count": 298,
        "sum_ns": 5561094582,
        "mean_ms": 18.6613912147651,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          190,
          108,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.723083
      },
      "callback_to_copied": {
        "count": 298,
        "sum_ns": 6151948816,
        "mean_ms": 20.64412354362416,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          140,
          158,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.206352
      },
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29799878577,
        "mean_ms": 99.99959254026845,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          7,
          39,
          99,
          148,
          5,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 166.845312
      },
      "selected_to_submitted": {
        "count": 298,
        "sum_ns": 17806061546,
        "mean_ms": 59.75188438255034,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          47,
          231,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 104.815064
      },
      "completed_queue_wait": {
        "count": 298,
        "sum_ns": 270227097,
        "mean_ms": 0.9068023389261745,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          214,
          68,
          15,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 7.956172
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.125111697707325,
    "first_redis_commit_s": 24.006831201724708,
    "maximum_sampled_silence_ms": 106.115372,
    "redis_commit_p95_bucket_upper_ms": 150
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.996261834166944,
    "completed": 299,
    "accepted": 299,
    "sent": 297,
    "late": 0,
    "completed_fps": 9.967908723193869,
    "accepted_fps": 9.967908723193869,
    "sent_fps": 9.901233748456786,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.939064974,
    "selected": 299,
    "infer_entries": 298,
    "infer_entry_fps": 9.953550662279945,
    "pre_infer_expired": 0,
    "diagnostic_missed": 7,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 2114435086,
        "mean_ms": 7.095419751677852,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          40,
          133,
          67,
          40,
          15,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.012645
      },
      "preprocess_start_to_preprocess_output": {
        "count": 297,
        "sum_ns": 2521875560,
        "mean_ms": 8.491163501683502,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          20,
          62,
          57,
          75,
          64,
          18,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 273.038281
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 6202872120,
        "mean_ms": 20.955649054054057,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          40,
          59,
          78,
          98,
          16,
          4,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 295.198767
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 4119627541,
        "mean_ms": 13.917660611486488,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          43,
          59,
          52,
          73,
          63,
          4,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 278.844967
      },
      "preprocess_output_to_infer_entry": {
        "count": 296,
        "sum_ns": 1615028618,
        "mean_ms": 5.456177763513514,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          103,
          43,
          52,
          47,
          36,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.059993
      },
      "preprocess_to_infer_output": {
        "count": 295,
        "sum_ns": 21140154719,
        "mean_ms": 71.66154142033899,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          129,
          73,
          46,
          25,
          16,
          3,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 560.63435
      },
      "preprocess_to_callback": {
        "count": 295,
        "sum_ns": 22887298243,
        "mean_ms": 77.58406184067796,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          94,
          98,
          44,
          32,
          19,
          5,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 566.169738
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 1762975980,
        "mean_ms": 5.955999932432433,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          60,
          120,
          61,
          44,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.961586
      },
      "copy_wall": {
        "count": 299,
        "sum_ns": 7485874548,
        "mean_ms": 25.03636972575251,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          3,
          73,
          115,
          95,
          7,
          1,
          0,
          0,
          0,
          5,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 447.216945
      },
      "copy_thread_cpu": {
        "count": 299,
        "sum_ns": 5406954413,
        "mean_ms": 18.08345957525084,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          3,
          111,
          157,
          23,
          0,
          0,
          0,
          0,
          0,
          5,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 392.400489
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 7700934303,
        "mean_ms": 26.016669942567567,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          63,
          121,
          96,
          9,
          1,
          0,
          0,
          0,
          5,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 447.942701
      },
      "accepted_interval": {
        "count": 299,
        "sum_ns": 29922358689,
        "mean_ms": 100.07477822408026,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          1,
          3,
          43,
          65,
          74,
          87,
          13,
          2,
          9,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 523.606895
      },
      "selected_to_submitted": {
        "count": 297,
        "sum_ns": 37638479645,
        "mean_ms": 126.7288876936027,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          7,
          61,
          67,
          96,
          28,
          28,
          7,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 607.391158
      },
      "completed_queue_wait": {
        "count": 299,
        "sum_ns": 8736388054,
        "mean_ms": 29.218689143812707,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          93,
          45,
          41,
          29,
          4,
          22,
          17,
          14,
          23,
          4,
          6,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 314.27704
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.668721463065594,
    "first_redis_commit_s": 19.77161428425461,
    "maximum_sampled_silence_ms": 75.136741,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.996261834166944,
    "completed": 300,
    "accepted": 300,
    "sent": 300,
    "late": 0,
    "completed_fps": 10.00124621056241,
    "accepted_fps": 10.00124621056241,
    "sent_fps": 10.00124621056241,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.880441266,
    "selected": 301,
    "infer_entries": 299,
    "infer_entry_fps": 10.006545664378208,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 1300553828,
        "mean_ms": 4.364274590604027,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          62,
          162,
          51,
          20,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.727684
      },
      "preprocess_start_to_preprocess_output": {
        "count": 298,
        "sum_ns": 1360476353,
        "mean_ms": 4.565356889261745,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          25,
          109,
          76,
          61,
          19,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 63.922111
      },
      "selected_to_preprocess": {
        "count": 298,
        "sum_ns": 3257084519,
        "mean_ms": 10.92981382214765,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          87,
          104,
          73,
          28,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.480954
      },
      "preprocess_start_to_preprocess": {
        "count": 298,
        "sum_ns": 1956530691,
        "mean_ms": 6.565539231543624,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          86,
          99,
          58,
          36,
          17,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 69.174265
      },
      "preprocess_output_to_infer_entry": {
        "count": 298,
        "sum_ns": 596054338,
        "mean_ms": 2.000182342281879,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          203,
          33,
          25,
          27,
          6,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 27.883217
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 13339721756,
        "mean_ms": 44.914888067340065,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          206,
          57,
          19,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 127.774063
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 14494395015,
        "mean_ms": 48.80267681818182,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          186,
          77,
          19,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 138.974228
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 1154673259,
        "mean_ms": 3.8877887508417506,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          37,
          198,
          52,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 14.956865
      },
      "copy_wall": {
        "count": 297,
        "sum_ns": 7356680102,
        "mean_ms": 24.769966673400674,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          99,
          189,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.51699
      },
      "copy_thread_cpu": {
        "count": 297,
        "sum_ns": 5524899272,
        "mean_ms": 18.60235445117845,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          198,
          99,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.04962
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 7627960632,
        "mean_ms": 25.77013727027027,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          83,
          202,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 89.005395
      },
      "accepted_interval": {
        "count": 297,
        "sum_ns": 29716271580,
        "mean_ms": 100.05478646464645,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          11,
          42,
          104,
          120,
          18,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 232.779219
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 25478856515,
        "mean_ms": 86.07721795608107,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          10,
          127,
          84,
          55,
          18,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 219.226018
      },
      "completed_queue_wait": {
        "count": 297,
        "sum_ns": 620906836,
        "mean_ms": 2.0905954074074073,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          147,
          52,
          71,
          20,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.660048
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.42959754401818,
    "first_redis_commit_s": 19.558911176398396,
    "maximum_sampled_silence_ms": 164.233194,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.996261834166944,
    "completed": 299,
    "accepted": 299,
    "sent": 299,
    "late": 0,
    "completed_fps": 9.967908723193869,
    "accepted_fps": 9.967908723193869,
    "sent_fps": 9.967908723193869,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.835248214,
    "selected": 301,
    "infer_entries": 297,
    "infer_entry_fps": 9.954668312785635,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "copy_wall": {
        "count": 297,
        "sum_ns": 6818525095,
        "mean_ms": 22.957996952861954,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          124,
          168,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 72.998105
      },
      "copy_thread_cpu": {
        "count": 297,
        "sum_ns": 5489928473,
        "mean_ms": 18.48460765319865,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          203,
          94,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.247297
      },
      "accepted_interval": {
        "count": 297,
        "sum_ns": 29691705051,
        "mean_ms": 99.97207087878787,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          16,
          48,
          98,
          109,
          24,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 290.62634
      },
      "completed_queue_wait": {
        "count": 297,
        "sum_ns": 382235221,
        "mean_ms": 1.286987276094276,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          181,
          75,
          34,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 8.898089
      },
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 1190222405,
        "mean_ms": 3.9940349161073825,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          64,
          178,
          39,
          16,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 37.745161
      },
      "preprocess_start_to_preprocess_output": {
        "count": 298,
        "sum_ns": 971556575,
        "mean_ms": 3.260256963087248,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          27,
          126,
          90,
          41,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.787431
      },
      "selected_to_preprocess": {
        "count": 297,
        "sum_ns": 2576046237,
        "mean_ms": 8.673556353535353,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          112,
          110,
          54,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.225505
      },
      "preprocess_start_to_preprocess": {
        "count": 297,
        "sum_ns": 1388289144,
        "mean_ms": 4.674374222222222,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          100,
          123,
          42,
          23,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.892539
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 437520000,
        "mean_ms": 1.473131313131313,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          227,
          31,
          17,
          12,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 21.190006
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 14477876670,
        "mean_ms": 48.7470594949495,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          202,
          53,
          17,
          12,
          1,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 330.49463
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 15598499905,
        "mean_ms": 52.5202017003367,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          193,
          62,
          21,
          13,
          2,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 335.186989
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 1120623235,
        "mean_ms": 3.7731422053872055,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          33,
          213,
          44,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.164242
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 7117682161,
        "mean_ms": 24.046223516891892,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          103,
          187,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 73.67135
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 25346563216,
        "mean_ms": 85.63028113513514,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          17,
          133,
          87,
          39,
          14,
          3,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 371.225928
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.429229367990047,
    "first_redis_commit_s": 21.930829759221524,
    "maximum_sampled_silence_ms": 143.628419,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 29.996261834166944,
    "completed": 299,
    "accepted": 299,
    "sent": 279,
    "late": 0,
    "completed_fps": 9.967908723193869,
    "accepted_fps": 9.967908723193869,
    "sent_fps": 9.301158975823041,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.031007212,
    "selected": 298,
    "infer_entries": 296,
    "infer_entry_fps": 9.856479268591507,
    "pre_infer_expired": 0,
    "diagnostic_missed": 10,
    "durations": {
      "accepted_interval": {
        "count": 299,
        "sum_ns": 30068832700,
        "mean_ms": 100.56465785953178,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          85,
          65,
          53,
          52,
          17,
          10,
          0,
          11,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 800.524495
      },
      "selected_to_preprocess_start": {
        "count": 297,
        "sum_ns": 3432673649,
        "mean_ms": 11.557823734006735,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          41,
          96,
          73,
          51,
          30,
          4,
          1,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 564.637337
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 4159190351,
        "mean_ms": 14.09895034237288,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          5,
          26,
          48,
          104,
          74,
          35,
          1,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 599.706649
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 9972706866,
        "mean_ms": 33.69157725,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          5,
          40,
          73,
          142,
          24,
          9,
          0,
          0,
          0,
          0,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 606.497989
      },
      "preprocess_start_to_preprocess": {
        "count": 294,
        "sum_ns": 6487650274,
        "mean_ms": 22.066837666666668,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          8,
          35,
          53,
          91,
          96,
          8,
          1,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 603.121227
      },
      "preprocess_output_to_infer_entry": {
        "count": 293,
        "sum_ns": 2318214031,
        "mean_ms": 7.9119932798634816,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          63,
          41,
          50,
          64,
          48,
          24,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 72.449964
      },
      "completed_queue_wait": {
        "count": 279,
        "sum_ns": 150942944641,
        "mean_ms": 541.0141384982079,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          4,
          0,
          0,
          1,
          0,
          2,
          0,
          2,
          7,
          5,
          25,
          80,
          152,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1001.230122
      },
      "copy_wall": {
        "count": 299,
        "sum_ns": 12050956892,
        "mean_ms": 40.30420365217391,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          13,
          74,
          172,
          29,
          5,
          1,
          0,
          0,
          0,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 767.36556
      },
      "copy_thread_cpu": {
        "count": 299,
        "sum_ns": 8145083462,
        "mean_ms": 27.24108181270903,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          24,
          150,
          119,
          2,
          0,
          0,
          0,
          0,
          1,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 602.498797
      },
      "preprocess_to_infer_output": {
        "count": 293,
        "sum_ns": 28791659137,
        "mean_ms": 98.26504824914676,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          76,
          68,
          48,
          62,
          23,
          10,
          1,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 697.678829
      },
      "preprocess_to_callback": {
        "count": 295,
        "sum_ns": 32089326360,
        "mean_ms": 108.77737749152543,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          46,
          76,
          57,
          65,
          29,
          15,
          2,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 706.913011
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 3124996272,
        "mean_ms": 10.557419837837838,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          14,
          89,
          103,
          68,
          18,
          3,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 430.509903
      },
      "callback_to_copied": {
        "count": 299,
        "sum_ns": 12359966274,
        "mean_ms": 41.33767984615385,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          12,
          72,
          170,
          34,
          5,
          1,
          0,
          0,
          0,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 767.969649
      },
      "selected_to_submitted": {
        "count": 299,
        "sum_ns": 55530444608,
        "mean_ms": 185.7205505284281,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          35,
          96,
          73,
          58,
          10,
          14,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 875.688087
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.32267033495009,
    "first_redis_commit_s": 22.42273849574849,
    "maximum_sampled_silence_ms": 111.894248,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 29.996261834166944,
    "completed": 296,
    "accepted": 296,
    "sent": 296,
    "late": 0,
    "completed_fps": 9.867896261088244,
    "accepted_fps": 9.867896261088244,
    "sent_fps": 9.867896261088244,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.854457843,
    "selected": 297,
    "infer_entries": 297,
    "infer_entry_fps": 9.948263055449786,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "copy_wall": {
        "count": 296,
        "sum_ns": 7084430268,
        "mean_ms": 23.93388604054054,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          105,
          186,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.456754
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5733182445,
        "mean_ms": 19.36885961148649,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          172,
          124,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.176149
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29826491066,
        "mean_ms": 100.76517252027027,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          18,
          46,
          100,
          108,
          20,
          3,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 311.587865
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 443821888,
        "mean_ms": 1.4993982702702704,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          180,
          59,
          47,
          6,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.044934
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 1138274940,
        "mean_ms": 3.845523445945946,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          89,
          146,
          45,
          14,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.234946
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 1058057045,
        "mean_ms": 3.586634050847458,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          47,
          106,
          73,
          47,
          20,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.049475
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 2711275301,
        "mean_ms": 9.15971385472973,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          112,
          97,
          59,
          28,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 57.809891
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 1573000361,
        "mean_ms": 5.314190408783785,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          2,
          96,
          106,
          42,
          40,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 33.530462
      },
      "preprocess_output_to_infer_entry": {
        "count": 295,
        "sum_ns": 495393274,
        "mean_ms": 1.679299233898305,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          226,
          20,
          23,
          17,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.34219
      },
      "preprocess_to_infer_output": {
        "count": 295,
        "sum_ns": 12518561051,
        "mean_ms": 42.43580017288136,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          14,
          210,
          46,
          15,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 190.291056
      },
      "preprocess_to_callback": {
        "count": 295,
        "sum_ns": 13819759763,
        "mean_ms": 46.84664326440678,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          198,
          61,
          19,
          10,
          2,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 208.493174
      },
      "infer_output_to_callback": {
        "count": 295,
        "sum_ns": 1301198712,
        "mean_ms": 4.4108430915254235,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          34,
          172,
          79,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 21.851646
      },
      "callback_to_copied": {
        "count": 295,
        "sum_ns": 7413161350,
        "mean_ms": 25.129360508474576,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          87,
          202,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.746462
      },
      "selected_to_submitted": {
        "count": 295,
        "sum_ns": 24168639493,
        "mean_ms": 81.92759150169492,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          24,
          138,
          71,
          48,
          11,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 275.078197
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.058295923750848,
    "first_redis_commit_s": 22.104469902813435,
    "maximum_sampled_silence_ms": 121.911421,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 2,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 29.996261834166944,
    "completed": 296,
    "accepted": 296,
    "sent": 296,
    "late": 0,
    "completed_fps": 9.867896261088244,
    "accepted_fps": 9.867896261088244,
    "sent_fps": 9.867896261088244,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.818107369,
    "selected": 296,
    "infer_entries": 296,
    "infer_entry_fps": 9.926854053377394,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29831283147,
        "mean_ms": 100.7813619831081,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          26,
          47,
          89,
          105,
          23,
          4,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 372.218111
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 372828423,
        "mean_ms": 1.2595554831081082,
        "p95_bucket_upper_ms": 5,
        "histogram": [
          194,
          56,
          38,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 10.82283
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 6943429895,
        "mean_ms": 23.457533429054056,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          110,
          183,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 73.768878
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5711615731,
        "mean_ms": 19.295999091216217,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          170,
          126,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.180235
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 1179734790,
        "mean_ms": 3.9855905067567567,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          76,
          156,
          45,
          17,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 36.880236
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 1148862035,
        "mean_ms": 3.8944475762711868,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          36,
          88,
          91,
          58,
          19,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.581955
      },
      "selected_to_preprocess": {
        "count": 296,
        "sum_ns": 2774767043,
        "mean_ms": 9.374212983108109,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          108,
          97,
          63,
          27,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 56.045285
      },
      "preprocess_start_to_preprocess": {
        "count": 296,
        "sum_ns": 1595032253,
        "mean_ms": 5.388622476351351,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          90,
          93,
          66,
          36,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.12297
      },
      "preprocess_output_to_infer_entry": {
        "count": 295,
        "sum_ns": 437310668,
        "mean_ms": 1.4824090440677966,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          218,
          25,
          26,
          22,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 13.073767
      },
      "preprocess_to_infer_output": {
        "count": 296,
        "sum_ns": 13549533999,
        "mean_ms": 45.775452699324326,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          17,
          197,
          46,
          18,
          11,
          2,
          5,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 298.481788
      },
      "preprocess_to_callback": {
        "count": 296,
        "sum_ns": 14829756329,
        "mean_ms": 50.10052813851351,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          8,
          188,
          58,
          24,
          11,
          2,
          4,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 305.887996
      },
      "infer_output_to_callback": {
        "count": 296,
        "sum_ns": 1280222330,
        "mean_ms": 4.325075439189189,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          0,
          25,
          193,
          67,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 17.339002
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 7305369365,
        "mean_ms": 24.680301908783782,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          93,
          197,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.432293
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 25096406863,
        "mean_ms": 84.78515832094595,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          26,
          133,
          68,
          50,
          13,
          3,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 359.264185
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.125111697707325,
    "first_redis_commit_s": 24.006831201724708,
    "maximum_sampled_silence_ms": 100.847388,
    "redis_commit_p95_bucket_upper_ms": 200
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 217,
    "accepted": 217,
    "sent": 184,
    "late": 0,
    "completed_fps": 7.233335284131277,
    "accepted_fps": 7.233335284131277,
    "sent_fps": 6.133334987466152,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.204821578,
    "selected": 216,
    "infer_entries": 216,
    "infer_entry_fps": 7.151176160475183,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "accepted_interval": {
        "count": 217,
        "sum_ns": 30361116237,
        "mean_ms": 139.91297805069124,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          19,
          30,
          51,
          55,
          29,
          20,
          4,
          8,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 722.236472
      },
      "copy_wall": {
        "count": 217,
        "sum_ns": 10847682142,
        "mean_ms": 49.98931862672811,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          26,
          103,
          49,
          28,
          8,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 420.513053
      },
      "copy_thread_cpu": {
        "count": 217,
        "sum_ns": 4430010495,
        "mean_ms": 20.4147949078341,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          12,
          134,
          69,
          1,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 362.22691
      },
      "completed_queue_wait": {
        "count": 216,
        "sum_ns": 15512989696,
        "mean_ms": 71.81939674074074,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          12,
          16,
          28,
          11,
          10,
          26,
          26,
          20,
          40,
          18,
          7,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 714.381834
      },
      "selected_to_preprocess_start": {
        "count": 211,
        "sum_ns": 2519116466,
        "mean_ms": 11.938940597156398,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          39,
          59,
          36,
          40,
          27,
          8,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 80.615403
      },
      "preprocess_start_to_preprocess_output": {
        "count": 211,
        "sum_ns": 3909124388,
        "mean_ms": 18.526655867298576,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          2,
          5,
          13,
          43,
          83,
          56,
          5,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 115.425025
      },
      "selected_to_preprocess": {
        "count": 214,
        "sum_ns": 10466928185,
        "mean_ms": 48.91087936915888,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          30,
          97,
          53,
          16,
          11,
          1,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 563.719621
      },
      "preprocess_start_to_preprocess": {
        "count": 211,
        "sum_ns": 7768227818,
        "mean_ms": 36.816245582938386,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          5,
          12,
          56,
          87,
          35,
          9,
          4,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 555.277589
      },
      "preprocess_output_to_infer_entry": {
        "count": 213,
        "sum_ns": 3891094243,
        "mean_ms": 18.26804808920188,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          23,
          16,
          18,
          51,
          47,
          45,
          10,
          2,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 541.438706
      },
      "preprocess_to_infer_output": {
        "count": 216,
        "sum_ns": 95329049437,
        "mean_ms": 441.338191837963,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          33,
          124,
          57,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1020.845913
      },
      "preprocess_to_callback": {
        "count": 215,
        "sum_ns": 98116121122,
        "mean_ms": 456.3540517302325,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          24,
          129,
          61,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1044.873158
      },
      "infer_output_to_callback": {
        "count": 215,
        "sum_ns": 3030510598,
        "mean_ms": 14.095398130232558,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          28,
          52,
          92,
          38,
          0,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 81.717962
      },
      "callback_to_copied": {
        "count": 215,
        "sum_ns": 11725224942,
        "mean_ms": 54.535929962790696,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          21,
          100,
          52,
          29,
          9,
          1,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 630.929557
      },
      "selected_to_submitted": {
        "count": 215,
        "sum_ns": 121417179255,
        "mean_ms": 564.7310663023256,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          99,
          103,
          11,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1124.543786
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.668721463065594,
    "first_redis_commit_s": 19.77161428425461,
    "maximum_sampled_silence_ms": 200.153311,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 299,
    "accepted": 299,
    "sent": 251,
    "late": 0,
    "completed_fps": 9.966669354632497,
    "accepted_fps": 9.966669354632497,
    "sent_fps": 8.366668923119589,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.175416701,
    "selected": 301,
    "infer_entries": 297,
    "infer_entry_fps": 9.842449002209058,
    "pre_infer_expired": 0,
    "diagnostic_missed": 10,
    "durations": {
      "copy_wall": {
        "count": 299,
        "sum_ns": 14182263956,
        "mean_ms": 47.43232092307692,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          16,
          172,
          72,
          30,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 117.2279
      },
      "copy_thread_cpu": {
        "count": 299,
        "sum_ns": 5909798000,
        "mean_ms": 19.765210702341136,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          150,
          149,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.092016
      },
      "accepted_interval": {
        "count": 299,
        "sum_ns": 30243264410,
        "mean_ms": 101.14804150501674,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          42,
          79,
          70,
          56,
          27,
          23,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 341.841371
      },
      "completed_queue_wait": {
        "count": 298,
        "sum_ns": 1074212867,
        "mean_ms": 3.60474116442953,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          62,
          73,
          93,
          51,
          18,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.642202
      },
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 2207677601,
        "mean_ms": 7.408314097315436,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          54,
          112,
          53,
          58,
          21,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.954384
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 3700390307,
        "mean_ms": 12.543695955932204,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          6,
          17,
          28,
          80,
          116,
          47,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 60.551555
      },
      "selected_to_preprocess": {
        "count": 295,
        "sum_ns": 8536636276,
        "mean_ms": 28.937750088135594,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          9,
          23,
          63,
          169,
          30,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 109.56145
      },
      "preprocess_start_to_preprocess": {
        "count": 294,
        "sum_ns": 6341085931,
        "mean_ms": 21.568319493197276,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          9,
          12,
          34,
          95,
          130,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.848586
      },
      "preprocess_output_to_infer_entry": {
        "count": 293,
        "sum_ns": 2620981495,
        "mean_ms": 8.945329334470989,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          63,
          26,
          45,
          57,
          77,
          23,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.245435
      },
      "preprocess_to_infer_output": {
        "count": 295,
        "sum_ns": 88009040362,
        "mean_ms": 298.33573004067796,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          7,
          30,
          123,
          133,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 506.118836
      },
      "preprocess_to_callback": {
        "count": 294,
        "sum_ns": 90846109866,
        "mean_ms": 309.0003736938776,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          5,
          27,
          114,
          144,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 518.936657
      },
      "infer_output_to_callback": {
        "count": 297,
        "sum_ns": 3192542600,
        "mean_ms": 10.749301683501683,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          56,
          94,
          116,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.980345
      },
      "callback_to_copied": {
        "count": 297,
        "sum_ns": 14536598683,
        "mean_ms": 48.94477671043771,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          170,
          74,
          32,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.503776
      },
      "selected_to_submitted": {
        "count": 297,
        "sum_ns": 115221497780,
        "mean_ms": 387.95117097643094,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          4,
          51,
          202,
          39,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 657.201765
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.42959754401818,
    "first_redis_commit_s": 19.558911176398396,
    "maximum_sampled_silence_ms": 69.711145,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 296,
    "accepted": 296,
    "sent": 240,
    "late": 0,
    "completed_fps": 9.86666932766294,
    "accepted_fps": 9.86666932766294,
    "sent_fps": 8.000002157564547,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.934296419,
    "selected": 297,
    "infer_entries": 295,
    "infer_entry_fps": 9.854916777424458,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "copy_wall": {
        "count": 296,
        "sum_ns": 12439878239,
        "mean_ms": 42.0266156722973,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          14,
          198,
          69,
          14,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.094209
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5761455203,
        "mean_ms": 19.46437568581081,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          147,
          149,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.765129
      },
      "accepted_interval": {
        "count": 296,
        "sum_ns": 29925868413,
        "mean_ms": 101.10090680067567,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          62,
          81,
          40,
          55,
          34,
          19,
          5,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 405.017782
      },
      "completed_queue_wait": {
        "count": 296,
        "sum_ns": 894214179,
        "mean_ms": 3.020993847972973,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          76,
          75,
          99,
          36,
          6,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 26.536333
      },
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 2032054932,
        "mean_ms": 6.865050445945945,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          61,
          106,
          59,
          56,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.901216
      },
      "preprocess_start_to_preprocess_output": {
        "count": 292,
        "sum_ns": 2900568282,
        "mean_ms": 9.933453020547946,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          6,
          28,
          48,
          94,
          86,
          30,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 46.060871
      },
      "selected_to_preprocess": {
        "count": 295,
        "sum_ns": 6801324680,
        "mean_ms": 23.055337898305083,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          12,
          34,
          96,
          139,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 88.916893
      },
      "preprocess_start_to_preprocess": {
        "count": 294,
        "sum_ns": 4733582662,
        "mean_ms": 16.100621299319727,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          11,
          28,
          66,
          98,
          87,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.221148
      },
      "preprocess_output_to_infer_entry": {
        "count": 293,
        "sum_ns": 1785919336,
        "mean_ms": 6.095287836177475,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          81,
          39,
          51,
          65,
          43,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.31504
      },
      "preprocess_to_infer_output": {
        "count": 293,
        "sum_ns": 91492045587,
        "mean_ms": 312.2595412525597,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          14,
          114,
          159,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 639.609145
      },
      "preprocess_to_callback": {
        "count": 294,
        "sum_ns": 94808552164,
        "mean_ms": 322.478068585034,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          12,
          101,
          174,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 645.006823
      },
      "infer_output_to_callback": {
        "count": 295,
        "sum_ns": 2885689844,
        "mean_ms": 9.78199947118644,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          62,
          117,
          83,
          26,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 44.108879
      },
      "callback_to_copied": {
        "count": 296,
        "sum_ns": 12976669335,
        "mean_ms": 43.840099104729724,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          194,
          68,
          19,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 104.867286
      },
      "selected_to_submitted": {
        "count": 296,
        "sum_ns": 115509857346,
        "mean_ms": 390.2360045472973,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          38,
          225,
          32,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 726.939526
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.429229367990047,
    "first_redis_commit_s": 21.930829759221524,
    "maximum_sampled_silence_ms": 295.266923,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 222,
    "accepted": 222,
    "sent": 197,
    "late": 0,
    "completed_fps": 7.400001995747205,
    "accepted_fps": 7.400001995747205,
    "sent_fps": 6.566668437667565,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.049809938,
    "selected": 221,
    "infer_entries": 219,
    "infer_entry_fps": 7.287899672305741,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "completed_queue_wait": {
        "count": 221,
        "sum_ns": 16997573452,
        "mean_ms": 76.9120970678733,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          17,
          25,
          28,
          21,
          10,
          27,
          23,
          10,
          20,
          14,
          15,
          7,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 656.97005
      },
      "selected_to_preprocess_start": {
        "count": 217,
        "sum_ns": 2976468570,
        "mean_ms": 13.716445023041475,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          41,
          67,
          24,
          45,
          33,
          5,
          0,
          1,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 472.660745
      },
      "preprocess_start_to_preprocess_output": {
        "count": 217,
        "sum_ns": 4220957881,
        "mean_ms": 19.45141880645161,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          5,
          12,
          57,
          81,
          49,
          7,
          1,
          1,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 434.598647
      },
      "copy_wall": {
        "count": 222,
        "sum_ns": 10077413410,
        "mean_ms": 45.3937540990991,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          6,
          35,
          102,
          51,
          19,
          8,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 466.023933
      },
      "copy_thread_cpu": {
        "count": 222,
        "sum_ns": 3997819358,
        "mean_ms": 18.008195306306305,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          25,
          148,
          47,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 330.740883
      },
      "accepted_interval": {
        "count": 222,
        "sum_ns": 30126455769,
        "mean_ms": 135.7047557162162,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          19,
          44,
          43,
          50,
          28,
          23,
          8,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 711.8869
      },
      "selected_to_preprocess": {
        "count": 216,
        "sum_ns": 10177929729,
        "mean_ms": 47.120045041666664,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          2,
          8,
          26,
          108,
          42,
          20,
          7,
          1,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 494.441552
      },
      "preprocess_start_to_preprocess": {
        "count": 215,
        "sum_ns": 7203293832,
        "mean_ms": 33.50369224186046,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          5,
          14,
          57,
          96,
          30,
          7,
          3,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 435.224993
      },
      "preprocess_output_to_infer_entry": {
        "count": 216,
        "sum_ns": 3061403173,
        "mean_ms": 14.173162837962963,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          41,
          7,
          22,
          44,
          50,
          47,
          2,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.997143
      },
      "preprocess_to_infer_output": {
        "count": 217,
        "sum_ns": 92326440473,
        "mean_ms": 425.4674676175115,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          29,
          136,
          48,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 845.288294
      },
      "preprocess_to_callback": {
        "count": 217,
        "sum_ns": 96145811148,
        "mean_ms": 443.06825413824885,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          27,
          132,
          57,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 869.703645
      },
      "infer_output_to_callback": {
        "count": 218,
        "sum_ns": 3694693531,
        "mean_ms": 16.948135463302755,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          37,
          52,
          85,
          40,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 429.988941
      },
      "callback_to_copied": {
        "count": 219,
        "sum_ns": 10218531456,
        "mean_ms": 46.65996098630137,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          5,
          29,
          102,
          54,
          20,
          8,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 466.480116
      },
      "selected_to_submitted": {
        "count": 219,
        "sum_ns": 118671869544,
        "mean_ms": 541.880682849315,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          98,
          114,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1022.514937
      }
    },
    "rejection_reasons": {},
    "first_response_s": 20.09501254884526,
    "first_redis_commit_s": 20.238557789009064,
    "maximum_sampled_silence_ms": 399.512393,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 303,
    "accepted": 303,
    "sent": 241,
    "late": 0,
    "completed_fps": 10.10000272392524,
    "accepted_fps": 10.10000272392524,
    "sent_fps": 8.033335499887732,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.321024442,
    "selected": 303,
    "infer_entries": 302,
    "infer_entry_fps": 9.960085635552485,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "copy_wall": {
        "count": 303,
        "sum_ns": 13468917149,
        "mean_ms": 44.45187177887789,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          25,
          178,
          75,
          17,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 126.989409
      },
      "copy_thread_cpu": {
        "count": 303,
        "sum_ns": 5829496469,
        "mean_ms": 19.239262273927395,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          160,
          143,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.736118
      },
      "accepted_interval": {
        "count": 303,
        "sum_ns": 30365894898,
        "mean_ms": 100.21747491089108,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          59,
          79,
          56,
          57,
          27,
          16,
          9,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 386.386164
      },
      "completed_queue_wait": {
        "count": 303,
        "sum_ns": 1151902918,
        "mean_ms": 3.801659795379538,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          81,
          70,
          91,
          38,
          12,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 38.688753
      },
      "selected_to_preprocess_start": {
        "count": 302,
        "sum_ns": 2140324474,
        "mean_ms": 7.087167132450331,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          71,
          102,
          54,
          55,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.482723
      },
      "preprocess_start_to_preprocess_output": {
        "count": 300,
        "sum_ns": 3437682910,
        "mean_ms": 11.458943033333334,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          7,
          19,
          49,
          95,
          100,
          28,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 233.509159
      },
      "selected_to_preprocess": {
        "count": 302,
        "sum_ns": 7683907468,
        "mean_ms": 25.443402211920528,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          12,
          47,
          85,
          134,
          20,
          2,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 236.734326
      },
      "preprocess_start_to_preprocess": {
        "count": 301,
        "sum_ns": 5523283012,
        "mean_ms": 18.349777448504984,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          14,
          30,
          55,
          100,
          95,
          4,
          1,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 234.241218
      },
      "preprocess_output_to_infer_entry": {
        "count": 301,
        "sum_ns": 2065680315,
        "mean_ms": 6.862725299003323,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          89,
          28,
          47,
          73,
          43,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.157448
      },
      "preprocess_to_infer_output": {
        "count": 302,
        "sum_ns": 88220151400,
        "mean_ms": 292.11970662251656,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          11,
          47,
          110,
          125,
          9,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 657.428682
      },
      "preprocess_to_callback": {
        "count": 302,
        "sum_ns": 91038986249,
        "mean_ms": 301.4535968509934,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          44,
          106,
          136,
          10,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 666.99076
      },
      "infer_output_to_callback": {
        "count": 303,
        "sum_ns": 2825588885,
        "mean_ms": 9.32537585808581,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          72,
          128,
          74,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.657796
      },
      "callback_to_copied": {
        "count": 303,
        "sum_ns": 13912236441,
        "mean_ms": 45.914971752475246,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          22,
          176,
          78,
          19,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 128.165895
      },
      "selected_to_submitted": {
        "count": 303,
        "sum_ns": 113307224134,
        "mean_ms": 373.95123476567653,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          72,
          188,
          35,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 752.907368
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.80268561001867,
    "first_redis_commit_s": 19.86646634573117,
    "maximum_sampled_silence_ms": 191.382389,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 302,
    "accepted": 302,
    "sent": 234,
    "late": 0,
    "completed_fps": 10.066669381602054,
    "accepted_fps": 10.066669381602054,
    "sent_fps": 7.800002103625433,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.266229775,
    "selected": 303,
    "infer_entries": 301,
    "infer_entry_fps": 9.945077475379076,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "copy_wall": {
        "count": 302,
        "sum_ns": 12231686152,
        "mean_ms": 40.50227202649007,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          16,
          214,
          54,
          16,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 104.503025
      },
      "copy_thread_cpu": {
        "count": 302,
        "sum_ns": 5793437875,
        "mean_ms": 19.18356912251656,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          166,
          136,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.317436
      },
      "accepted_interval": {
        "count": 302,
        "sum_ns": 30219773227,
        "mean_ms": 100.0654742615894,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          67,
          80,
          53,
          45,
          27,
          24,
          6,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 374.943001
      },
      "selected_to_preprocess_start": {
        "count": 302,
        "sum_ns": 1906234926,
        "mean_ms": 6.312036178807947,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          94,
          92,
          52,
          45,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.597204
      },
      "completed_queue_wait": {
        "count": 302,
        "sum_ns": 811998875,
        "mean_ms": 2.688737996688742,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          104,
          66,
          90,
          32,
          9,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.948901
      },
      "preprocess_start_to_preprocess_output": {
        "count": 300,
        "sum_ns": 2935091759,
        "mean_ms": 9.783639196666668,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          4,
          31,
          40,
          105,
          97,
          22,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 58.030168
      },
      "selected_to_preprocess": {
        "count": 301,
        "sum_ns": 6980015250,
        "mean_ms": 23.189419435215946,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          13,
          42,
          85,
          147,
          11,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 96.986238
      },
      "preprocess_start_to_preprocess": {
        "count": 300,
        "sum_ns": 5061729839,
        "mean_ms": 16.87243279666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          10,
          34,
          58,
          106,
          83,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 73.027144
      },
      "preprocess_output_to_infer_entry": {
        "count": 300,
        "sum_ns": 2129777907,
        "mean_ms": 7.09925969,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          97,
          26,
          41,
          67,
          44,
          22,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 60.696891
      },
      "preprocess_to_infer_output": {
        "count": 297,
        "sum_ns": 93899756968,
        "mean_ms": 316.1607978720539,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          17,
          116,
          157,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 541.501927
      },
      "preprocess_to_callback": {
        "count": 297,
        "sum_ns": 96578671723,
        "mean_ms": 325.18071287205385,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          106,
          169,
          6,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 545.764023
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 2688210127,
        "mean_ms": 8.990669321070234,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          8,
          75,
          117,
          82,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.678419
      },
      "callback_to_copied": {
        "count": 299,
        "sum_ns": 12563612762,
        "mean_ms": 42.01877177926421,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          14,
          207,
          59,
          17,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 105.698828
      },
      "selected_to_submitted": {
        "count": 299,
        "sum_ns": 116854273461,
        "mean_ms": 390.81696809698997,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          52,
          214,
          33,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 612.424843
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.804386627860367,
    "first_redis_commit_s": 22.491312731988728,
    "maximum_sampled_silence_ms": 295.128667,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "pose",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 242,
    "accepted": 242,
    "sent": 201,
    "late": 0,
    "completed_fps": 8.066668842210918,
    "accepted_fps": 8.066668842210918,
    "sent_fps": 6.700001806960308,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.090687143,
    "selected": 242,
    "infer_entries": 241,
    "infer_entry_fps": 8.009122518694754,
    "pre_infer_expired": 0,
    "diagnostic_missed": 6,
    "durations": {
      "completed_queue_wait": {
        "count": 243,
        "sum_ns": 14463732743,
        "mean_ms": 59.5215339218107,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          15,
          18,
          33,
          18,
          16,
          36,
          31,
          22,
          36,
          12,
          2,
          2,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 571.854439
      },
      "selected_to_preprocess_start": {
        "count": 240,
        "sum_ns": 2649468662,
        "mean_ms": 11.039452758333333,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          54,
          79,
          23,
          36,
          42,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 87.913873
      },
      "preprocess_start_to_preprocess_output": {
        "count": 237,
        "sum_ns": 4073175268,
        "mean_ms": 17.18639353586498,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          7,
          19,
          48,
          91,
          64,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.326126
      },
      "selected_to_preprocess": {
        "count": 241,
        "sum_ns": 11319348108,
        "mean_ms": 46.96824941078838,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          2,
          2,
          42,
          118,
          49,
          14,
          12,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 544.681343
      },
      "preprocess_start_to_preprocess": {
        "count": 239,
        "sum_ns": 8608403033,
        "mean_ms": 36.01842273221757,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          7,
          14,
          55,
          122,
          23,
          10,
          4,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 542.948569
      },
      "preprocess_output_to_infer_entry": {
        "count": 238,
        "sum_ns": 4452748593,
        "mean_ms": 18.70902770168067,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          25,
          14,
          30,
          49,
          59,
          50,
          3,
          6,
          0,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 506.647404
      },
      "copy_wall": {
        "count": 242,
        "sum_ns": 10701810818,
        "mean_ms": 44.222358752066114,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          15,
          35,
          148,
          25,
          11,
          3,
          0,
          0,
          4,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 525.562735
      },
      "copy_thread_cpu": {
        "count": 242,
        "sum_ns": 4767757633,
        "mean_ms": 19.70147782231405,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          48,
          154,
          35,
          0,
          0,
          0,
          0,
          1,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 366.798672
      },
      "accepted_interval": {
        "count": 242,
        "sum_ns": 29999987836,
        "mean_ms": 123.96689188429752,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          1,
          1,
          37,
          63,
          39,
          42,
          24,
          17,
          13,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 613.822723
      },
      "preprocess_to_infer_output": {
        "count": 238,
        "sum_ns": 101265527750,
        "mean_ms": 425.4854107142857,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          31,
          150,
          54,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 815.08398
      },
      "preprocess_to_callback": {
        "count": 238,
        "sum_ns": 104374757146,
        "mean_ms": 438.54939977310926,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          24,
          144,
          67,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 835.729118
      },
      "infer_output_to_callback": {
        "count": 239,
        "sum_ns": 3114868388,
        "mean_ms": 13.032922125523012,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          9,
          37,
          61,
          86,
          43,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.09161
      },
      "callback_to_copied": {
        "count": 238,
        "sum_ns": 10855766795,
        "mean_ms": 45.612465525210084,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          14,
          26,
          146,
          33,
          10,
          4,
          0,
          0,
          4,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 526.056211
      },
      "selected_to_submitted": {
        "count": 238,
        "sum_ns": 127034397709,
        "mean_ms": 533.7579735672269,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          118,
          116,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1116.883086
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.832174353301525,
    "first_redis_commit_s": 21.98553210636601,
    "maximum_sampled_silence_ms": 371.27477,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "fire",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 301,
    "accepted": 301,
    "sent": 247,
    "late": 0,
    "completed_fps": 10.033336039278868,
    "accepted_fps": 10.033336039278868,
    "sent_fps": 8.233335553826846,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.84757906,
    "selected": 301,
    "infer_entries": 298,
    "infer_entry_fps": 9.984059323570479,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "accepted_interval": {
        "count": 297,
        "sum_ns": 29836077370,
        "mean_ms": 100.45817296296298,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          1,
          1,
          80,
          59,
          47,
          48,
          25,
          31,
          5,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 349.242796
      },
      "completed_queue_wait": {
        "count": 297,
        "sum_ns": 708501369,
        "mean_ms": 2.3855264949494948,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          122,
          73,
          66,
          25,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 25.359826
      },
      "selected_to_preprocess_start": {
        "count": 298,
        "sum_ns": 1704769890,
        "mean_ms": 5.72070432885906,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          1,
          96,
          94,
          51,
          48,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.980305
      },
      "preprocess_start_to_preprocess_output": {
        "count": 297,
        "sum_ns": 2727758799,
        "mean_ms": 9.184373060606061,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          9,
          35,
          52,
          82,
          100,
          18,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.686501
      },
      "selected_to_preprocess": {
        "count": 298,
        "sum_ns": 6004201321,
        "mean_ms": 20.148326580536914,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          31,
          48,
          80,
          130,
          6,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 91.055997
      },
      "preprocess_start_to_preprocess": {
        "count": 298,
        "sum_ns": 4299431431,
        "mean_ms": 14.427622251677851,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          22,
          47,
          53,
          93,
          79,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.010052
      },
      "preprocess_output_to_infer_entry": {
        "count": 297,
        "sum_ns": 1546189349,
        "mean_ms": 5.206024744107744,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          116,
          36,
          27,
          60,
          50,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.015783
      },
      "copy_wall": {
        "count": 296,
        "sum_ns": 10891934371,
        "mean_ms": 36.7970755777027,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          47,
          194,
          44,
          9,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 192.009079
      },
      "copy_thread_cpu": {
        "count": 296,
        "sum_ns": 5473416195,
        "mean_ms": 18.491270929054057,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          186,
          110,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.744853
      },
      "preprocess_to_infer_output": {
        "count": 295,
        "sum_ns": 82658682281,
        "mean_ms": 280.19892298644066,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          8,
          37,
          133,
          113,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 532.617687
      },
      "preprocess_to_callback": {
        "count": 295,
        "sum_ns": 84902345358,
        "mean_ms": 287.8045605355932,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          8,
          28,
          130,
          125,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 539.256154
      },
      "infer_output_to_callback": {
        "count": 295,
        "sum_ns": 2243663077,
        "mean_ms": 7.605637549152543,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          17,
          100,
          104,
          65,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.962076
      },
      "callback_to_copied": {
        "count": 295,
        "sum_ns": 11263147854,
        "mean_ms": 38.18016221694915,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          38,
          196,
          48,
          11,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 192.808672
      },
      "selected_to_submitted": {
        "count": 295,
        "sum_ns": 102377960396,
        "mean_ms": 347.04393354576274,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          9,
          86,
          186,
          12,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 596.638486
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.69765533413738,
    "first_redis_commit_s": 21.832689728122205,
    "maximum_sampled_silence_ms": 90.011744,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 29.999991909135133,
    "completed": 297,
    "accepted": 297,
    "sent": 238,
    "late": 0,
    "completed_fps": 9.900002669986126,
    "accepted_fps": 9.900002669986126,
    "sent_fps": 7.933335472918175,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "configure_to_last_child_snapshot",
    "diagnostic_seconds": 29.660925985,
    "selected": 299,
    "infer_entries": 296,
    "infer_entry_fps": 9.979459176348435,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 296,
        "sum_ns": 1636509601,
        "mean_ms": 5.528748652027026,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          0,
          91,
          105,
          47,
          39,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.869466
      },
      "preprocess_start_to_preprocess_output": {
        "count": 295,
        "sum_ns": 3078174694,
        "mean_ms": 10.434490488135593,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          6,
          32,
          40,
          85,
          102,
          29,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.462926
      },
      "selected_to_preprocess": {
        "count": 295,
        "sum_ns": 6693886538,
        "mean_ms": 22.69114080677966,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          20,
          45,
          73,
          145,
          8,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 121.532962
      },
      "preprocess_start_to_preprocess": {
        "count": 295,
        "sum_ns": 5067945953,
        "mean_ms": 17.17947780677966,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          16,
          41,
          41,
          93,
          100,
          0,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 112.820161
      },
      "preprocess_output_to_infer_entry": {
        "count": 295,
        "sum_ns": 1989771259,
        "mean_ms": 6.7449873186440685,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          92,
          23,
          36,
          81,
          50,
          10,
          2,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 111.996228
      },
      "copy_wall": {
        "count": 295,
        "sum_ns": 11475341173,
        "mean_ms": 38.89946160338983,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          25,
          204,
          55,
          9,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 122.218855
      },
      "copy_thread_cpu": {
        "count": 295,
        "sum_ns": 5605299139,
        "mean_ms": 19.001014030508472,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          168,
          127,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.971571
      },
      "accepted_interval": {
        "count": 295,
        "sum_ns": 29614988032,
        "mean_ms": 100.38978993898306,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          74,
          66,
          44,
          48,
          30,
          27,
          4,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 405.67686
      },
      "completed_queue_wait": {
        "count": 295,
        "sum_ns": 728844142,
        "mean_ms": 2.4706581084745762,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          120,
          63,
          78,
          22,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 19.701421
      },
      "preprocess_to_infer_output": {
        "count": 293,
        "sum_ns": 92251395247,
        "mean_ms": 314.8511783174061,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          11,
          119,
          158,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 546.481862
      },
      "preprocess_to_callback": {
        "count": 291,
        "sum_ns": 93985711419,
        "mean_ms": 322.97495332989695,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          112,
          163,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 566.945162
      },
      "infer_output_to_callback": {
        "count": 291,
        "sum_ns": 2542344011,
        "mean_ms": 8.736577357388315,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          6,
          86,
          115,
          66,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 39.300933
      },
      "callback_to_copied": {
        "count": 290,
        "sum_ns": 11613217977,
        "mean_ms": 40.04557923103448,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          23,
          196,
          59,
          10,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 122.696234
      },
      "selected_to_submitted": {
        "count": 292,
        "sum_ns": 113205215742,
        "mean_ms": 387.6890950068493,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          41,
          229,
          22,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 651.402312
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.59981249924749,
    "first_redis_commit_s": 24.518204372376204,
    "maximum_sampled_silence_ms": 129.271391,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 220,
    "accepted": 220,
    "sent": 182,
    "late": 0,
    "completed_fps": 7.333335311100834,
    "accepted_fps": 7.333335311100834,
    "sent_fps": 6.066668302819781,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.725741222,
    "selected": 219,
    "infer_entries": 218,
    "infer_entry_fps": 7.333711155322121,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "copy_wall": {
        "count": 220,
        "sum_ns": 12496740044,
        "mean_ms": 56.80336383636364,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          18,
          123,
          48,
          20,
          6,
          0,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 760.819889
      },
      "copy_thread_cpu": {
        "count": 220,
        "sum_ns": 5700357490,
        "mean_ms": 25.910715863636362,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          10,
          136,
          71,
          0,
          0,
          0,
          0,
          0,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 567.099803
      },
      "accepted_interval": {
        "count": 220,
        "sum_ns": 29770220991,
        "mean_ms": 135.31918632272726,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          29,
          50,
          33,
          45,
          25,
          25,
          4,
          8,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 800.515019
      },
      "completed_queue_wait": {
        "count": 205,
        "sum_ns": 60364932197,
        "mean_ms": 294.4630838878049,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          3,
          8,
          5,
          1,
          2,
          8,
          10,
          16,
          29,
          16,
          32,
          32,
          43,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 994.605063
      },
      "selected_to_preprocess_start": {
        "count": 217,
        "sum_ns": 2875191419,
        "mean_ms": 13.249730041474654,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          35,
          71,
          24,
          36,
          40,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 96.845584
      },
      "preprocess_start_to_preprocess_output": {
        "count": 216,
        "sum_ns": 5232698855,
        "mean_ms": 24.225457662037037,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          20,
          52,
          71,
          62,
          8,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 695.926463
      },
      "selected_to_preprocess": {
        "count": 217,
        "sum_ns": 12166050551,
        "mean_ms": 56.06474908294931,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          4,
          21,
          118,
          47,
          19,
          2,
          1,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 760.542646
      },
      "preprocess_start_to_preprocess": {
        "count": 216,
        "sum_ns": 9268390838,
        "mean_ms": 42.90921684259259,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          5,
          19,
          55,
          101,
          26,
          5,
          1,
          0,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 741.893371
      },
      "preprocess_output_to_infer_entry": {
        "count": 216,
        "sum_ns": 4041332232,
        "mean_ms": 18.709871444444445,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          25,
          13,
          27,
          60,
          50,
          32,
          5,
          2,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 671.026515
      },
      "preprocess_to_infer_output": {
        "count": 217,
        "sum_ns": 98748835481,
        "mean_ms": 455.0637579769585,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          32,
          132,
          46,
          7,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1117.001973
      },
      "preprocess_to_callback": {
        "count": 217,
        "sum_ns": 102904344850,
        "mean_ms": 474.21357073732725,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          22,
          136,
          51,
          8,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1139.478583
      },
      "infer_output_to_callback": {
        "count": 218,
        "sum_ns": 4165709234,
        "mean_ms": 19.10875795412844,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          11,
          53,
          100,
          45,
          6,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 639.197301
      },
      "callback_to_copied": {
        "count": 219,
        "sum_ns": 12838268798,
        "mean_ms": 58.622231954337906,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          12,
          122,
          51,
          23,
          6,
          0,
          0,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 761.879147
      },
      "selected_to_submitted": {
        "count": 219,
        "sum_ns": 129738480968,
        "mean_ms": 592.4131551050228,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          96,
          99,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1382.478664
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.32267033495009,
    "first_redis_commit_s": 22.42273849574849,
    "maximum_sampled_silence_ms": 724.814061,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 300,
    "accepted": 300,
    "sent": 246,
    "late": 0,
    "completed_fps": 10.000002696955683,
    "accepted_fps": 10.000002696955683,
    "sent_fps": 8.20000221150366,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.093107608,
    "selected": 301,
    "infer_entries": 301,
    "infer_entry_fps": 10.002290355682032,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "accepted_interval": {
        "count": 300,
        "sum_ns": 29994806862,
        "mean_ms": 99.98268954000001,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          72,
          67,
          45,
          54,
          36,
          23,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 408.375503
      },
      "completed_queue_wait": {
        "count": 300,
        "sum_ns": 796844955,
        "mean_ms": 2.6561498500000003,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          94,
          72,
          98,
          24,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.321762
      },
      "selected_to_preprocess_start": {
        "count": 300,
        "sum_ns": 2192176118,
        "mean_ms": 7.307253726666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          58,
          114,
          50,
          56,
          22,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.985315
      },
      "preprocess_start_to_preprocess_output": {
        "count": 300,
        "sum_ns": 2881673974,
        "mean_ms": 9.605579913333335,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          5,
          32,
          38,
          103,
          99,
          23,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.402211
      },
      "selected_to_preprocess": {
        "count": 301,
        "sum_ns": 6743001545,
        "mean_ms": 22.40199848837209,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          18,
          41,
          97,
          127,
          18,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.152867
      },
      "preprocess_start_to_preprocess": {
        "count": 300,
        "sum_ns": 4518665842,
        "mean_ms": 15.062219473333332,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          15,
          34,
          60,
          106,
          82,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 67.822736
      },
      "preprocess_output_to_infer_entry": {
        "count": 301,
        "sum_ns": 1656639552,
        "mean_ms": 5.503785887043189,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          113,
          21,
          37,
          71,
          52,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.411737
      },
      "copy_wall": {
        "count": 300,
        "sum_ns": 12058315201,
        "mean_ms": 40.19438400333333,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          25,
          198,
          54,
          18,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 118.965013
      },
      "copy_thread_cpu": {
        "count": 300,
        "sum_ns": 5926387491,
        "mean_ms": 19.75462497,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          145,
          155,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.034606
      },
      "preprocess_to_infer_output": {
        "count": 299,
        "sum_ns": 84508245077,
        "mean_ms": 282.6362711605351,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          13,
          41,
          115,
          127,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 492.925435
      },
      "preprocess_to_callback": {
        "count": 299,
        "sum_ns": 87061115133,
        "mean_ms": 291.17429810367895,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          9,
          35,
          118,
          134,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 499.419414
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 2552870056,
        "mean_ms": 8.538026943143812,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          5,
          91,
          114,
          72,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.880367
      },
      "callback_to_copied": {
        "count": 300,
        "sum_ns": 12492096001,
        "mean_ms": 41.64032000333333,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          24,
          192,
          61,
          17,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.799384
      },
      "selected_to_submitted": {
        "count": 300,
        "sum_ns": 106793206372,
        "mean_ms": 355.97735457333334,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          12,
          85,
          182,
          20,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 653.298351
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.058295923750848,
    "first_redis_commit_s": 22.104469902813435,
    "maximum_sampled_silence_ms": 114.615663,
    "redis_commit_p95_bucket_upper_ms": 300
  },
  {
    "level": 4,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 29.999991909135133,
    "completed": 298,
    "accepted": 298,
    "sent": 238,
    "late": 0,
    "completed_fps": 9.933336012309312,
    "accepted_fps": 9.933336012309312,
    "sent_fps": 7.933335472918175,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.054100325,
    "selected": 301,
    "infer_entries": 299,
    "infer_entry_fps": 9.948725690227429,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 301,
        "sum_ns": 1992542933,
        "mean_ms": 6.61974396345515,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          77,
          108,
          47,
          52,
          16,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.445828
      },
      "preprocess_start_to_preprocess_output": {
        "count": 300,
        "sum_ns": 3087297696,
        "mean_ms": 10.29099232,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          29,
          43,
          76,
          128,
          23,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.209346
      },
      "selected_to_preprocess": {
        "count": 299,
        "sum_ns": 7030020160,
        "mean_ms": 23.511773110367894,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          20,
          36,
          77,
          157,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 82.938773
      },
      "preprocess_start_to_preprocess": {
        "count": 299,
        "sum_ns": 5050253712,
        "mean_ms": 16.890480642140467,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          18,
          36,
          42,
          89,
          111,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 74.602682
      },
      "preprocess_output_to_infer_entry": {
        "count": 298,
        "sum_ns": 1981407677,
        "mean_ms": 6.64901905033557,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          98,
          23,
          28,
          77,
          60,
          10,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 60.034202
      },
      "copy_wall": {
        "count": 298,
        "sum_ns": 11958190205,
        "mean_ms": 40.12815505033557,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          26,
          198,
          58,
          10,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 126.762886
      },
      "copy_thread_cpu": {
        "count": 298,
        "sum_ns": 5918035516,
        "mean_ms": 19.85917958389262,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          146,
          152,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.518791
      },
      "accepted_interval": {
        "count": 298,
        "sum_ns": 29912846334,
        "mean_ms": 100.37867897315436,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          57,
          77,
          50,
          57,
          34,
          19,
          3,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 354.341662
      },
      "completed_queue_wait": {
        "count": 299,
        "sum_ns": 777939097,
        "mean_ms": 2.601803,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          110,
          79,
          67,
          33,
          7,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 23.539857
      },
      "preprocess_to_infer_output": {
        "count": 298,
        "sum_ns": 93077331257,
        "mean_ms": 312.3400377751678,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          15,
          127,
          152,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 523.158686
      },
      "preprocess_to_callback": {
        "count": 298,
        "sum_ns": 95925077806,
        "mean_ms": 321.8962342483221,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          9,
          119,
          166,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 532.188687
      },
      "infer_output_to_callback": {
        "count": 299,
        "sum_ns": 2851033388,
        "mean_ms": 9.535228722408027,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          77,
          116,
          72,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.356494
      },
      "callback_to_copied": {
        "count": 298,
        "sum_ns": 12393482823,
        "mean_ms": 41.58886853355705,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          23,
          195,
          62,
          11,
          7,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 127.362403
      },
      "selected_to_submitted": {
        "count": 298,
        "sum_ns": 115752286739,
        "mean_ms": 388.4304924127517,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          40,
          226,
          32,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 627.845318
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.125111697707325,
    "first_redis_commit_s": 24.006831201724708,
    "maximum_sampled_silence_ms": 141.980495,
    "redis_commit_p95_bucket_upper_ms": 500
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 119,
    "accepted": 119,
    "sent": 43,
    "late": 0,
    "completed_fps": 3.9666626988039204,
    "accepted_fps": 3.9666626988039204,
    "sent_fps": 1.433331899567803,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.863370853,
    "selected": 119,
    "infer_entries": 120,
    "infer_entry_fps": 4.018300565957212,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "completed_queue_wait": {
        "count": 120,
        "sum_ns": 3509239491,
        "mean_ms": 29.243662425,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          21,
          19,
          23,
          12,
          11,
          12,
          5,
          3,
          11,
          0,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 271.509388
      },
      "selected_to_preprocess_start": {
        "count": 116,
        "sum_ns": 2418128960,
        "mean_ms": 20.845939310344825,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          19,
          26,
          17,
          21,
          26,
          5,
          0,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 693.413638
      },
      "copy_wall": {
        "count": 119,
        "sum_ns": 9873035330,
        "mean_ms": 82.96668344537815,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          11,
          47,
          23,
          18,
          12,
          3,
          1,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 855.841172
      },
      "copy_thread_cpu": {
        "count": 119,
        "sum_ns": 3972373815,
        "mean_ms": 33.38129256302521,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          3,
          75,
          37,
          0,
          0,
          0,
          0,
          0,
          3,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 503.036522
      },
      "accepted_interval": {
        "count": 119,
        "sum_ns": 29791049926,
        "mean_ms": 250.34495736134454,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          1,
          9,
          8,
          22,
          20,
          28,
          20,
          9,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1104.77636
      },
      "preprocess_start_to_preprocess_output": {
        "count": 115,
        "sum_ns": 2200138504,
        "mean_ms": 19.131639165217393,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          4,
          16,
          49,
          35,
          2,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.899694
      },
      "selected_to_preprocess": {
        "count": 118,
        "sum_ns": 7891425199,
        "mean_ms": 66.87648473728814,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          1,
          1,
          10,
          53,
          22,
          12,
          9,
          6,
          3,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 736.322201
      },
      "preprocess_start_to_preprocess": {
        "count": 117,
        "sum_ns": 5454606014,
        "mean_ms": 46.62056422222222,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          2,
          1,
          4,
          21,
          58,
          11,
          6,
          7,
          6,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 214.909213
      },
      "preprocess_output_to_infer_entry": {
        "count": 115,
        "sum_ns": 3106045396,
        "mean_ms": 27.009090399999998,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          10,
          7,
          13,
          24,
          22,
          21,
          8,
          2,
          4,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 212.978068
      },
      "preprocess_to_infer_output": {
        "count": 117,
        "sum_ns": 99181165823,
        "mean_ms": 847.7022719914529,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          85,
          29,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1720.288119
      },
      "preprocess_to_callback": {
        "count": 117,
        "sum_ns": 101149736963,
        "mean_ms": 864.5276663504274,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          85,
          31,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1733.549353
      },
      "infer_output_to_callback": {
        "count": 117,
        "sum_ns": 1968571140,
        "mean_ms": 16.82539435897436,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          14,
          35,
          38,
          26,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 109.657135
      },
      "callback_to_copied": {
        "count": 117,
        "sum_ns": 9999549463,
        "mean_ms": 85.46623472649573,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          0,
          10,
          44,
          21,
          23,
          11,
          3,
          1,
          0,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 857.013236
      },
      "selected_to_submitted": {
        "count": 117,
        "sum_ns": 119594104677,
        "mean_ms": 1022.1718348461538,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          65,
          52,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1939.13926
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.668721463065594,
    "first_redis_commit_s": 19.77161428425461,
    "maximum_sampled_silence_ms": 1010.444017,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 162,
    "accepted": 162,
    "sent": 55,
    "late": 0,
    "completed_fps": 5.399994598371723,
    "accepted_fps": 5.399994598371723,
    "sent_fps": 1.83333149944719,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.897022071,
    "selected": 161,
    "infer_entries": 162,
    "infer_entry_fps": 5.418599873100385,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 161,
        "sum_ns": 1967314446,
        "mean_ms": 12.219344385093168,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          29,
          37,
          25,
          35,
          32,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.588217
      },
      "preprocess_start_to_preprocess_output": {
        "count": 161,
        "sum_ns": 2533406135,
        "mean_ms": 15.735441832298138,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          3,
          5,
          42,
          69,
          41,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.597885
      },
      "selected_to_preprocess": {
        "count": 161,
        "sum_ns": 7018645047,
        "mean_ms": 43.59406861490683,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          3,
          19,
          89,
          27,
          18,
          3,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 202.991553
      },
      "preprocess_start_to_preprocess": {
        "count": 161,
        "sum_ns": 5051330601,
        "mean_ms": 31.374724229813665,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          4,
          10,
          38,
          86,
          17,
          3,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 196.048096
      },
      "preprocess_output_to_infer_entry": {
        "count": 161,
        "sum_ns": 2517924466,
        "mean_ms": 15.639282397515528,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          22,
          7,
          13,
          37,
          52,
          20,
          6,
          1,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 186.612969
      },
      "accepted_interval": {
        "count": 162,
        "sum_ns": 29832863277,
        "mean_ms": 184.15347701851852,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          8,
          11,
          20,
          34,
          24,
          43,
          20,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 517.110151
      },
      "completed_queue_wait": {
        "count": 162,
        "sum_ns": 465851887,
        "mean_ms": 2.8756289320987656,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          53,
          36,
          48,
          19,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.857364
      },
      "copy_wall": {
        "count": 162,
        "sum_ns": 10419462277,
        "mean_ms": 64.31766837654321,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          56,
          46,
          34,
          19,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 197.415091
      },
      "copy_thread_cpu": {
        "count": 162,
        "sum_ns": 3225488038,
        "mean_ms": 19.910419987654322,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          81,
          81,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.828812
      },
      "preprocess_to_infer_output": {
        "count": 161,
        "sum_ns": 104972351987,
        "mean_ms": 652.0021862546583,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          27,
          130,
          4,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1129.449047
      },
      "preprocess_to_callback": {
        "count": 161,
        "sum_ns": 107128523326,
        "mean_ms": 665.3945548198757,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          27,
          130,
          4,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1134.691401
      },
      "infer_output_to_callback": {
        "count": 161,
        "sum_ns": 2156171339,
        "mean_ms": 13.39236856521739,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          21,
          58,
          51,
          30,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.867507
      },
      "callback_to_copied": {
        "count": 161,
        "sum_ns": 10940755505,
        "mean_ms": 67.95500313664596,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          52,
          46,
          36,
          20,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 208.973513
      },
      "selected_to_submitted": {
        "count": 161,
        "sum_ns": 125323487406,
        "mean_ms": 778.4067540745342,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          136,
          16,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1250.997379
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.42959754401818,
    "first_redis_commit_s": 19.558911176398396,
    "maximum_sampled_silence_ms": 149.326969,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 5,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 151,
    "accepted": 151,
    "sent": 51,
    "late": 0,
    "completed_fps": 5.033328298482285,
    "accepted_fps": 5.033328298482285,
    "sent_fps": 1.6999982994873943,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.79202078,
    "selected": 151,
    "infer_entries": 151,
    "infer_entry_fps": 5.068471223052094,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 151,
        "sum_ns": 9268326882,
        "mean_ms": 61.37964822516556,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          64,
          33,
          30,
          16,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 161.81776
      },
      "copy_thread_cpu": {
        "count": 151,
        "sum_ns": 3003133538,
        "mean_ms": 19.88830157615894,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          71,
          80,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.94123
      },
      "accepted_interval": {
        "count": 151,
        "sum_ns": 29769266951,
        "mean_ms": 197.14746325165564,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          16,
          17,
          21,
          30,
          36,
          25,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 561.701442
      },
      "completed_queue_wait": {
        "count": 152,
        "sum_ns": 388257550,
        "mean_ms": 2.5543259868421053,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          48,
          49,
          34,
          17,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 21.391548
      },
      "selected_to_preprocess_start": {
        "count": 151,
        "sum_ns": 1752928681,
        "mean_ms": 11.60879921192053,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          21,
          36,
          28,
          39,
          25,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 66.176686
      },
      "preprocess_start_to_preprocess_output": {
        "count": 150,
        "sum_ns": 1974503283,
        "mean_ms": 13.163355220000001,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          6,
          10,
          43,
          70,
          20,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 48.827755
      },
      "selected_to_preprocess": {
        "count": 151,
        "sum_ns": 5707428587,
        "mean_ms": 37.79754031125828,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          31,
          85,
          19,
          12,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 148.738574
      },
      "preprocess_start_to_preprocess": {
        "count": 151,
        "sum_ns": 3954499906,
        "mean_ms": 26.18874109933775,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          3,
          17,
          53,
          62,
          8,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 132.899033
      },
      "preprocess_output_to_infer_entry": {
        "count": 150,
        "sum_ns": 1954078906,
        "mean_ms": 13.027192706666668,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          30,
          12,
          16,
          37,
          33,
          12,
          6,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 122.392349
      },
      "preprocess_to_infer_output": {
        "count": 151,
        "sum_ns": 106751202697,
        "mean_ms": 706.9616072649006,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          21,
          118,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1324.220779
      },
      "preprocess_to_callback": {
        "count": 151,
        "sum_ns": 108769328022,
        "mean_ms": 720.3266756423841,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          20,
          118,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1348.341046
      },
      "infer_output_to_callback": {
        "count": 151,
        "sum_ns": 2018125325,
        "mean_ms": 13.365068377483444,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          21,
          43,
          62,
          24,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.455902
      },
      "callback_to_copied": {
        "count": 151,
        "sum_ns": 9851124122,
        "mean_ms": 65.23923259602648,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          57,
          35,
          33,
          18,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 162.632713
      },
      "selected_to_submitted": {
        "count": 151,
        "sum_ns": 124543603107,
        "mean_ms": 824.7920735562914,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          114,
          31,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1418.181423
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.429229367990047,
    "first_redis_commit_s": 21.930829759221524,
    "maximum_sampled_silence_ms": 225.382831,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 133,
    "accepted": 133,
    "sent": 51,
    "late": 0,
    "completed_fps": 4.433328898663205,
    "accepted_fps": 4.433328898663205,
    "sent_fps": 1.6999982994873943,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.069745529,
    "selected": 133,
    "infer_entries": 131,
    "infer_entry_fps": 4.356538364239245,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "accepted_interval": {
        "count": 133,
        "sum_ns": 30133777764,
        "mean_ms": 226.56975762406014,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          7,
          10,
          7,
          22,
          24,
          26,
          30,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1023.788953
      },
      "selected_to_preprocess_start": {
        "count": 132,
        "sum_ns": 1615282709,
        "mean_ms": 12.236990219696969,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          27,
          37,
          18,
          25,
          21,
          3,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.594487
      },
      "completed_queue_wait": {
        "count": 132,
        "sum_ns": 2763665439,
        "mean_ms": 20.936859386363636,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          35,
          23,
          31,
          15,
          7,
          11,
          2,
          1,
          2,
          2,
          1,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 591.535311
      },
      "preprocess_start_to_preprocess_output": {
        "count": 131,
        "sum_ns": 1866494502,
        "mean_ms": 14.248049633587787,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          6,
          13,
          38,
          49,
          22,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 80.968988
      },
      "selected_to_preprocess": {
        "count": 130,
        "sum_ns": 6642287406,
        "mean_ms": 51.09451850769231,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          4,
          20,
          64,
          26,
          7,
          4,
          2,
          1,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 540.822912
      },
      "preprocess_start_to_preprocess": {
        "count": 130,
        "sum_ns": 5115354948,
        "mean_ms": 39.348884215384615,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          1,
          8,
          13,
          29,
          54,
          14,
          3,
          4,
          1,
          1,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 539.163148
      },
      "preprocess_output_to_infer_entry": {
        "count": 129,
        "sum_ns": 3144451233,
        "mean_ms": 24.375590953488373,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          32,
          2,
          16,
          19,
          31,
          15,
          7,
          2,
          1,
          2,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 525.246439
      },
      "copy_wall": {
        "count": 133,
        "sum_ns": 5552999262,
        "mean_ms": 41.75187415037594,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          4,
          26,
          64,
          20,
          10,
          8,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 141.972301
      },
      "copy_thread_cpu": {
        "count": 133,
        "sum_ns": 1813604990,
        "mean_ms": 13.636127744360902,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          1,
          27,
          90,
          15,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.082585
      },
      "preprocess_to_infer_output": {
        "count": 132,
        "sum_ns": 106796956343,
        "mean_ms": 809.0678510833334,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          102,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1557.196227
      },
      "preprocess_to_callback": {
        "count": 130,
        "sum_ns": 106475395390,
        "mean_ms": 819.041503,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          97,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1563.619091
      },
      "infer_output_to_callback": {
        "count": 131,
        "sum_ns": 1279005404,
        "mean_ms": 9.763400030534351,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          8,
          26,
          56,
          32,
          8,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.886687
      },
      "callback_to_copied": {
        "count": 131,
        "sum_ns": 5783979273,
        "mean_ms": 44.15251353435115,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          23,
          67,
          19,
          10,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 144.100747
      },
      "selected_to_submitted": {
        "count": 132,
        "sum_ns": 121208078292,
        "mean_ms": 918.2430173636363,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          93,
          39,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1704.023687
      }
    },
    "rejection_reasons": {},
    "first_response_s": 20.09501254884526,
    "first_redis_commit_s": 20.238557789009064,
    "maximum_sampled_silence_ms": 262.626373,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 173,
    "accepted": 173,
    "sent": 56,
    "late": 0,
    "completed_fps": 5.766660898261161,
    "accepted_fps": 5.766660898261161,
    "sent_fps": 1.866664799437139,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.631625231,
    "selected": 172,
    "infer_entries": 173,
    "infer_entry_fps": 5.647757789388188,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 172,
        "sum_ns": 1471762133,
        "mean_ms": 8.556756587209303,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          55,
          39,
          28,
          28,
          21,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.15549
      },
      "copy_wall": {
        "count": 173,
        "sum_ns": 10465462740,
        "mean_ms": 60.494004277456646,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          67,
          58,
          21,
          17,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 185.840316
      },
      "copy_thread_cpu": {
        "count": 173,
        "sum_ns": 3246072587,
        "mean_ms": 18.763425358381504,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          108,
          65,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.588805
      },
      "accepted_interval": {
        "count": 173,
        "sum_ns": 30696729616,
        "mean_ms": 177.4377434450867,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          8,
          12,
          18,
          39,
          35,
          44,
          16,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 533.16546
      },
      "completed_queue_wait": {
        "count": 173,
        "sum_ns": 592508211,
        "mean_ms": 3.4249029537572255,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          56,
          41,
          44,
          20,
          7,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.653901
      },
      "preprocess_start_to_preprocess_output": {
        "count": 173,
        "sum_ns": 2488546759,
        "mean_ms": 14.384663346820808,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          9,
          6,
          47,
          72,
          39,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 49.345523
      },
      "selected_to_preprocess": {
        "count": 173,
        "sum_ns": 5908187380,
        "mean_ms": 34.15137213872832,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          3,
          3,
          31,
          103,
          28,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 85.073714
      },
      "preprocess_start_to_preprocess": {
        "count": 173,
        "sum_ns": 4429694360,
        "mean_ms": 25.605169710982658,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          3,
          3,
          13,
          56,
          85,
          12,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.573062
      },
      "preprocess_output_to_infer_entry": {
        "count": 173,
        "sum_ns": 1941147601,
        "mean_ms": 11.22050636416185,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          34,
          9,
          20,
          38,
          41,
          27,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 73.038522
      },
      "preprocess_to_infer_output": {
        "count": 172,
        "sum_ns": 109741008424,
        "mean_ms": 638.029118744186,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          36,
          135,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1022.116695
      },
      "preprocess_to_callback": {
        "count": 172,
        "sum_ns": 111564338592,
        "mean_ms": 648.6298755348837,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          32,
          138,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1058.81797
      },
      "infer_output_to_callback": {
        "count": 172,
        "sum_ns": 1823330168,
        "mean_ms": 10.600756790697675,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          37,
          75,
          38,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.638614
      },
      "callback_to_copied": {
        "count": 173,
        "sum_ns": 10874505173,
        "mean_ms": 62.85841140462428,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          63,
          62,
          23,
          18,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 189.817695
      },
      "selected_to_submitted": {
        "count": 173,
        "sum_ns": 129435621269,
        "mean_ms": 748.1827819017341,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          9,
          149,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1226.118867
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.80268561001867,
    "first_redis_commit_s": 19.86646634573117,
    "maximum_sampled_silence_ms": 284.064609,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 6,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 155,
    "accepted": 155,
    "sent": 51,
    "late": 0,
    "completed_fps": 5.166661498442081,
    "accepted_fps": 5.166661498442081,
    "sent_fps": 1.6999982994873943,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.264538156,
    "selected": 154,
    "infer_entries": 154,
    "infer_entry_fps": 5.088463574305997,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "completed_queue_wait": {
        "count": 154,
        "sum_ns": 454011172,
        "mean_ms": 2.9481244935064934,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          58,
          36,
          33,
          20,
          5,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.156163
      },
      "selected_to_preprocess_start": {
        "count": 153,
        "sum_ns": 1402645987,
        "mean_ms": 9.16762083006536,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          29,
          40,
          32,
          32,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.290747
      },
      "preprocess_start_to_preprocess_output": {
        "count": 153,
        "sum_ns": 2138409826,
        "mean_ms": 13.976534810457517,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          10,
          38,
          73,
          28,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.553085
      },
      "selected_to_preprocess": {
        "count": 153,
        "sum_ns": 5361099243,
        "mean_ms": 35.039864333333334,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          3,
          2,
          28,
          90,
          24,
          4,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 178.555256
      },
      "preprocess_start_to_preprocess": {
        "count": 153,
        "sum_ns": 3958453256,
        "mean_ms": 25.872243503267974,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          3,
          3,
          10,
          56,
          67,
          10,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 166.93143
      },
      "preprocess_output_to_infer_entry": {
        "count": 153,
        "sum_ns": 1820043430,
        "mean_ms": 11.895708692810457,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          32,
          11,
          15,
          44,
          28,
          14,
          6,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 145.681868
      },
      "copy_wall": {
        "count": 155,
        "sum_ns": 8762771780,
        "mean_ms": 56.53401148387097,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          9,
          75,
          34,
          22,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.321663
      },
      "copy_thread_cpu": {
        "count": 155,
        "sum_ns": 2943452028,
        "mean_ms": 18.990013083870966,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          83,
          72,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 29.037175
      },
      "accepted_interval": {
        "count": 155,
        "sum_ns": 30401637433,
        "mean_ms": 196.1395963419355,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          6,
          12,
          12,
          28,
          43,
          26,
          25,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 663.713712
      },
      "preprocess_to_infer_output": {
        "count": 154,
        "sum_ns": 109813326712,
        "mean_ms": 713.073550077922,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          20,
          122,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1245.674086
      },
      "preprocess_to_callback": {
        "count": 154,
        "sum_ns": 111563218888,
        "mean_ms": 724.4364862857143,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          15,
          127,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1271.446118
      },
      "infer_output_to_callback": {
        "count": 154,
        "sum_ns": 1749892176,
        "mean_ms": 11.362936207792208,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          36,
          51,
          40,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.754537
      },
      "callback_to_copied": {
        "count": 154,
        "sum_ns": 9080292358,
        "mean_ms": 58.96293738961039,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          66,
          42,
          23,
          14,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 194.539908
      },
      "selected_to_submitted": {
        "count": 154,
        "sum_ns": 126302800489,
        "mean_ms": 820.1480551233766,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          128,
          22,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1478.795959
      }
    },
    "rejection_reasons": {},
    "first_response_s": 19.804386627860367,
    "first_redis_commit_s": 22.491312731988728,
    "maximum_sampled_silence_ms": 286.655847,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 135,
    "accepted": 135,
    "sent": 53,
    "late": 0,
    "completed_fps": 4.499995498643103,
    "accepted_fps": 4.499995498643103,
    "sent_fps": 1.7666648994672922,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.450798474,
    "selected": 135,
    "infer_entries": 134,
    "infer_entry_fps": 4.400541421415077,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 135,
        "sum_ns": 7251936112,
        "mean_ms": 53.71804527407408,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          20,
          69,
          24,
          9,
          9,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 912.544988
      },
      "copy_thread_cpu": {
        "count": 135,
        "sum_ns": 2729575197,
        "mean_ms": 20.219075533333335,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          15,
          105,
          13,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 401.762007
      },
      "accepted_interval": {
        "count": 135,
        "sum_ns": 30925037023,
        "mean_ms": 229.07434831851853,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          11,
          13,
          22,
          23,
          31,
          21,
          9,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1154.751439
      },
      "completed_queue_wait": {
        "count": 135,
        "sum_ns": 1414839840,
        "mean_ms": 10.480295111111111,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          42,
          25,
          32,
          9,
          8,
          10,
          5,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 144.563127
      },
      "selected_to_preprocess_start": {
        "count": 135,
        "sum_ns": 1213936941,
        "mean_ms": 8.99212548888889,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          44,
          36,
          17,
          18,
          17,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 691.526836
      },
      "preprocess_start_to_preprocess_output": {
        "count": 135,
        "sum_ns": 2986469826,
        "mean_ms": 22.12199871111111,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          4,
          32,
          59,
          35,
          1,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 866.440178
      },
      "selected_to_preprocess": {
        "count": 134,
        "sum_ns": 5926671783,
        "mean_ms": 44.22889390298507,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          17,
          90,
          11,
          7,
          6,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 870.620435
      },
      "preprocess_start_to_preprocess": {
        "count": 134,
        "sum_ns": 4728634369,
        "mean_ms": 35.288316186567165,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          6,
          39,
          75,
          7,
          2,
          3,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 867.320458
      },
      "preprocess_output_to_infer_entry": {
        "count": 134,
        "sum_ns": 1753618575,
        "mean_ms": 13.08670578358209,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          22,
          5,
          15,
          36,
          36,
          15,
          1,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 352.682828
      },
      "preprocess_to_infer_output": {
        "count": 134,
        "sum_ns": 111616846646,
        "mean_ms": 832.9615421343284,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          97,
          31,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1739.038739
      },
      "preprocess_to_callback": {
        "count": 134,
        "sum_ns": 113112599667,
        "mean_ms": 844.1238781119403,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          93,
          35,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1751.636573
      },
      "infer_output_to_callback": {
        "count": 135,
        "sum_ns": 1500072915,
        "mean_ms": 11.111651222222221,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          35,
          45,
          34,
          18,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.590026
      },
      "callback_to_copied": {
        "count": 135,
        "sum_ns": 7473560591,
        "mean_ms": 55.35970808148148,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          16,
          71,
          23,
          12,
          9,
          0,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 919.394841
      },
      "selected_to_submitted": {
        "count": 135,
        "sum_ns": 129400680018,
        "mean_ms": 958.5235556888889,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          79,
          52,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1997.16695
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.832174353301525,
    "first_redis_commit_s": 21.98553210636601,
    "maximum_sampled_silence_ms": 786.722328,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 175,
    "accepted": 175,
    "sent": 53,
    "late": 0,
    "completed_fps": 5.833327498241059,
    "accepted_fps": 5.833327498241059,
    "sent_fps": 1.7666648994672922,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.291429831,
    "selected": 175,
    "infer_entries": 174,
    "infer_entry_fps": 5.74419896884266,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "accepted_interval": {
        "count": 175,
        "sum_ns": 30266242754,
        "mean_ms": 172.94995859428573,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          9,
          18,
          18,
          37,
          35,
          38,
          20,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 454.559813
      },
      "completed_queue_wait": {
        "count": 175,
        "sum_ns": 419236540,
        "mean_ms": 2.3956373714285717,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          83,
          26,
          39,
          23,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 18.386778
      },
      "selected_to_preprocess_start": {
        "count": 175,
        "sum_ns": 1258175136,
        "mean_ms": 7.189572205714286,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          59,
          35,
          31,
          37,
          12,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.1183
      },
      "preprocess_start_to_preprocess_output": {
        "count": 174,
        "sum_ns": 2650742909,
        "mean_ms": 15.234154649425287,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          0,
          6,
          39,
          87,
          41,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 56.658679
      },
      "selected_to_preprocess": {
        "count": 174,
        "sum_ns": 6050735855,
        "mean_ms": 34.774343994252874,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          0,
          27,
          122,
          19,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 123.209683
      },
      "preprocess_start_to_preprocess": {
        "count": 174,
        "sum_ns": 4806304459,
        "mean_ms": 27.622439419540232,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          2,
          10,
          60,
          90,
          7,
          3,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.293907
      },
      "preprocess_output_to_infer_entry": {
        "count": 174,
        "sum_ns": 2155561550,
        "mean_ms": 12.388284770114943,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          26,
          13,
          17,
          48,
          45,
          19,
          3,
          1,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 110.649561
      },
      "copy_wall": {
        "count": 175,
        "sum_ns": 8728463654,
        "mean_ms": 49.876935165714286,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          12,
          95,
          37,
          20,
          11,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 132.010233
      },
      "copy_thread_cpu": {
        "count": 175,
        "sum_ns": 3114853019,
        "mean_ms": 17.799160108571428,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          119,
          56,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.578756
      },
      "preprocess_to_infer_output": {
        "count": 174,
        "sum_ns": 111234163801,
        "mean_ms": 639.276803454023,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          38,
          134,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1072.070393
      },
      "preprocess_to_callback": {
        "count": 174,
        "sum_ns": 113069147564,
        "mean_ms": 649.8226871494253,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          31,
          141,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1090.249669
      },
      "infer_output_to_callback": {
        "count": 175,
        "sum_ns": 1842393966,
        "mean_ms": 10.52796552,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          38,
          66,
          47,
          18,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.661956
      },
      "callback_to_copied": {
        "count": 175,
        "sum_ns": 9046807617,
        "mean_ms": 51.69604352571429,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          95,
          41,
          18,
          14,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 132.620366
      },
      "selected_to_submitted": {
        "count": 175,
        "sum_ns": 129109488078,
        "mean_ms": 737.7685033028572,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          155,
          8,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1207.846452
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.69765533413738,
    "first_redis_commit_s": 21.832689728122205,
    "maximum_sampled_silence_ms": 281.400683,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 8,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 159,
    "accepted": 159,
    "sent": 47,
    "late": 0,
    "completed_fps": 5.299994698401877,
    "accepted_fps": 5.299994698401877,
    "sent_fps": 1.5666650995275988,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.285997871,
    "selected": 159,
    "infer_entries": 159,
    "infer_entry_fps": 5.249950841218562,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 158,
        "sum_ns": 1251362813,
        "mean_ms": 7.920017803797468,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          53,
          32,
          25,
          32,
          16,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.702453
      },
      "preprocess_start_to_preprocess_output": {
        "count": 158,
        "sum_ns": 2280415740,
        "mean_ms": 14.433011012658227,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          0,
          8,
          35,
          87,
          27,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.978983
      },
      "selected_to_preprocess": {
        "count": 158,
        "sum_ns": 5237368948,
        "mean_ms": 33.14790473417722,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          1,
          30,
          105,
          17,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 88.078566
      },
      "preprocess_start_to_preprocess": {
        "count": 158,
        "sum_ns": 3986006135,
        "mean_ms": 25.22788693037975,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          2,
          12,
          58,
          73,
          10,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.064736
      },
      "preprocess_output_to_infer_entry": {
        "count": 158,
        "sum_ns": 1705590395,
        "mean_ms": 10.79487591772152,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          36,
          6,
          16,
          45,
          35,
          14,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 75.800644
      },
      "copy_wall": {
        "count": 159,
        "sum_ns": 7619129897,
        "mean_ms": 47.91905595597484,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          92,
          41,
          16,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 118.917456
      },
      "copy_thread_cpu": {
        "count": 159,
        "sum_ns": 2860596922,
        "mean_ms": 17.991175610062893,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          107,
          52,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 28.748739
      },
      "accepted_interval": {
        "count": 159,
        "sum_ns": 30335776264,
        "mean_ms": 190.79104568553458,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          10,
          5,
          13,
          34,
          31,
          46,
          19,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 612.471528
      },
      "completed_queue_wait": {
        "count": 159,
        "sum_ns": 291806122,
        "mean_ms": 1.8352586289308175,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          72,
          37,
          35,
          15,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 8.656727
      },
      "preprocess_to_infer_output": {
        "count": 158,
        "sum_ns": 112973086183,
        "mean_ms": 715.0195328037975,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          17,
          129,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1329.818529
      },
      "preprocess_to_callback": {
        "count": 158,
        "sum_ns": 114580994226,
        "mean_ms": 725.1961659873417,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          15,
          131,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1334.150484
      },
      "infer_output_to_callback": {
        "count": 158,
        "sum_ns": 1607908043,
        "mean_ms": 10.176633183544304,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          39,
          58,
          43,
          16,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 38.24769
      },
      "callback_to_copied": {
        "count": 158,
        "sum_ns": 7963741985,
        "mean_ms": 50.40343028481013,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          86,
          47,
          15,
          6,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.564449
      },
      "selected_to_submitted": {
        "count": 158,
        "sum_ns": 127917397751,
        "mean_ms": 809.6037832341773,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          7,
          131,
          20,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1412.651986
      }
    },
    "rejection_reasons": {},
    "first_response_s": 21.59981249924749,
    "first_redis_commit_s": 24.518204372376204,
    "maximum_sampled_silence_ms": 145.620745,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 129,
    "accepted": 129,
    "sent": 46,
    "late": 0,
    "completed_fps": 4.29999569870341,
    "accepted_fps": 4.29999569870341,
    "sent_fps": 1.5333317995376499,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.498304339,
    "selected": 129,
    "infer_entries": 127,
    "infer_entry_fps": 4.164165934877813,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 128,
        "sum_ns": 1846138563,
        "mean_ms": 14.4229575234375,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          24,
          31,
          16,
          29,
          24,
          1,
          1,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 214.90196
      },
      "copy_wall": {
        "count": 129,
        "sum_ns": 7964753740,
        "mean_ms": 61.74227705426357,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          15,
          51,
          29,
          20,
          6,
          2,
          2,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 594.901361
      },
      "copy_thread_cpu": {
        "count": 129,
        "sum_ns": 3204650370,
        "mean_ms": 24.84225093023256,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          7,
          82,
          37,
          1,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 437.339419
      },
      "accepted_interval": {
        "count": 129,
        "sum_ns": 30487101748,
        "mean_ms": 236.3341220775194,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          9,
          3,
          11,
          18,
          24,
          32,
          22,
          10,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 882.199664
      },
      "preprocess_start_to_preprocess_output": {
        "count": 127,
        "sum_ns": 2846252571,
        "mean_ms": 22.411437566929134,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          2,
          6,
          28,
          39,
          41,
          6,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 128.579176
      },
      "completed_queue_wait": {
        "count": 129,
        "sum_ns": 3189080527,
        "mean_ms": 24.721554472868217,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          30,
          21,
          28,
          9,
          4,
          14,
          6,
          7,
          8,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 245.888093
      },
      "selected_to_preprocess": {
        "count": 126,
        "sum_ns": 8386763399,
        "mean_ms": 66.56161427777778,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          0,
          2,
          6,
          59,
          26,
          14,
          10,
          5,
          3,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 664.912507
      },
      "preprocess_start_to_preprocess": {
        "count": 126,
        "sum_ns": 6548764764,
        "mean_ms": 51.974323523809524,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          8,
          23,
          55,
          17,
          10,
          7,
          2,
          2,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 652.756597
      },
      "preprocess_output_to_infer_entry": {
        "count": 126,
        "sum_ns": 3715512821,
        "mean_ms": 29.488196992063493,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          18,
          10,
          13,
          23,
          26,
          17,
          7,
          6,
          2,
          1,
          2,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 644.591824
      },
      "preprocess_to_infer_output": {
        "count": 126,
        "sum_ns": 106896629469,
        "mean_ms": 848.3859481666666,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          4,
          95,
          27,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1723.34958
      },
      "preprocess_to_callback": {
        "count": 126,
        "sum_ns": 108955180831,
        "mean_ms": 864.7236573888888,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          96,
          28,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1741.848127
      },
      "infer_output_to_callback": {
        "count": 128,
        "sum_ns": 2079138148,
        "mean_ms": 16.24326678125,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          3,
          21,
          31,
          47,
          17,
          5,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 88.446359
      },
      "callback_to_copied": {
        "count": 128,
        "sum_ns": 8178620682,
        "mean_ms": 63.895474078125,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          2,
          11,
          53,
          29,
          17,
          10,
          2,
          2,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 595.35558
      },
      "selected_to_submitted": {
        "count": 128,
        "sum_ns": 127484144077,
        "mean_ms": 995.9698756015625,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          73,
          54,
          1,
          0
        ],
        "diagnostic_window_max_ms": 2079.15467
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.894174412824214,
    "first_redis_commit_s": 24.062348559033126,
    "maximum_sampled_silence_ms": 221.763485,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 177,
    "accepted": 177,
    "sent": 57,
    "late": 0,
    "completed_fps": 5.899994098220957,
    "accepted_fps": 5.899994098220957,
    "sent_fps": 1.899998099427088,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.827040376,
    "selected": 176,
    "infer_entries": 177,
    "infer_entry_fps": 5.741712400578068,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 175,
        "sum_ns": 1948659918,
        "mean_ms": 11.135199531428572,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          37,
          43,
          21,
          39,
          33,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.262543
      },
      "preprocess_start_to_preprocess_output": {
        "count": 175,
        "sum_ns": 2713103240,
        "mean_ms": 15.503447085714287,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          3,
          5,
          41,
          87,
          36,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.987051
      },
      "accepted_interval": {
        "count": 177,
        "sum_ns": 30886254384,
        "mean_ms": 174.49861233898307,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          14,
          13,
          47,
          34,
          52,
          12,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 459.110253
      },
      "completed_queue_wait": {
        "count": 177,
        "sum_ns": 549966497,
        "mean_ms": 3.1071553502824862,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          70,
          41,
          42,
          16,
          6,
          1,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 89.922979
      },
      "selected_to_preprocess": {
        "count": 175,
        "sum_ns": 7428331035,
        "mean_ms": 42.44760591428571,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          25,
          97,
          34,
          12,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 151.201146
      },
      "preprocess_start_to_preprocess": {
        "count": 175,
        "sum_ns": 5479671117,
        "mean_ms": 31.312406382857144,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          1,
          2,
          12,
          45,
          92,
          13,
          7,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 142.083513
      },
      "preprocess_output_to_infer_entry": {
        "count": 175,
        "sum_ns": 2766567877,
        "mean_ms": 15.808959297142858,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          28,
          13,
          10,
          43,
          46,
          19,
          12,
          1,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 125.807185
      },
      "copy_wall": {
        "count": 177,
        "sum_ns": 9913696683,
        "mean_ms": 56.009585779661016,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          8,
          86,
          43,
          26,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 165.425106
      },
      "copy_thread_cpu": {
        "count": 177,
        "sum_ns": 3390780368,
        "mean_ms": 19.156951231638416,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          98,
          79,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 31.605131
      },
      "preprocess_to_infer_output": {
        "count": 175,
        "sum_ns": 111883962521,
        "mean_ms": 639.3369286914285,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          36,
          137,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1026.805657
      },
      "preprocess_to_callback": {
        "count": 175,
        "sum_ns": 114115186950,
        "mean_ms": 652.0867825714286,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          30,
          142,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1040.555911
      },
      "infer_output_to_callback": {
        "count": 175,
        "sum_ns": 2231224429,
        "mean_ms": 12.749853880000002,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          31,
          52,
          62,
          28,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.45583
      },
      "callback_to_copied": {
        "count": 176,
        "sum_ns": 10404173649,
        "mean_ms": 59.11462300568182,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          80,
          43,
          28,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 178.534778
      },
      "selected_to_submitted": {
        "count": 176,
        "sum_ns": 133002355931,
        "mean_ms": 755.6952041534091,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          160,
          8,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1159.968348
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.650979397818446,
    "first_redis_commit_s": 23.905873668845743,
    "maximum_sampled_silence_ms": 222.583043,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 9,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 161,
    "accepted": 161,
    "sent": 50,
    "late": 0,
    "completed_fps": 5.366661298381774,
    "accepted_fps": 5.366661298381774,
    "sent_fps": 1.6666649994974454,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.680954272,
    "selected": 161,
    "infer_entries": 161,
    "infer_entry_fps": 5.247555163136876,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 161,
        "sum_ns": 1432458666,
        "mean_ms": 8.897258795031055,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          45,
          45,
          14,
          36,
          20,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 61.514582
      },
      "preprocess_start_to_preprocess_output": {
        "count": 161,
        "sum_ns": 2338198435,
        "mean_ms": 14.522971645962734,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          1,
          37,
          92,
          28,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 38.957208
      },
      "selected_to_preprocess": {
        "count": 161,
        "sum_ns": 6157314296,
        "mean_ms": 38.244188173913045,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          33,
          98,
          21,
          3,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 150.739678
      },
      "preprocess_start_to_preprocess": {
        "count": 161,
        "sum_ns": 4724855630,
        "mean_ms": 29.34692937888199,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          0,
          6,
          61,
          76,
          10,
          2,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 148.833809
      },
      "preprocess_output_to_infer_entry": {
        "count": 161,
        "sum_ns": 2386657195,
        "mean_ms": 14.823957732919256,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          29,
          7,
          10,
          50,
          37,
          18,
          4,
          1,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 123.510571
      },
      "copy_wall": {
        "count": 161,
        "sum_ns": 8173691615,
        "mean_ms": 50.76827090062112,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          97,
          38,
          14,
          6,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 197.386189
      },
      "copy_thread_cpu": {
        "count": 161,
        "sum_ns": 3147246494,
        "mean_ms": 19.548114869565218,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          92,
          69,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.35243
      },
      "accepted_interval": {
        "count": 161,
        "sum_ns": 30684158011,
        "mean_ms": 190.58483236645964,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          11,
          19,
          30,
          33,
          42,
          18,
          3,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 656.74661
      },
      "completed_queue_wait": {
        "count": 161,
        "sum_ns": 369706025,
        "mean_ms": 2.296310714285714,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          71,
          37,
          33,
          18,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.222024
      },
      "preprocess_to_infer_output": {
        "count": 161,
        "sum_ns": 113062107819,
        "mean_ms": 702.2491168881987,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          132,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1312.53452
      },
      "preprocess_to_callback": {
        "count": 161,
        "sum_ns": 115169819123,
        "mean_ms": 715.340491447205,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          130,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1318.510363
      },
      "infer_output_to_callback": {
        "count": 161,
        "sum_ns": 2107711304,
        "mean_ms": 13.09137455900621,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          21,
          52,
          58,
          29,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 42.424724
      },
      "callback_to_copied": {
        "count": 161,
        "sum_ns": 8693701795,
        "mean_ms": 53.99814779503106,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          86,
          45,
          16,
          8,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 206.985915
      },
      "selected_to_submitted": {
        "count": 161,
        "sum_ns": 130271824392,
        "mean_ms": 809.1417664099379,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          8,
          132,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1376.619122
      }
    },
    "rejection_reasons": {},
    "first_response_s": 23.573962600901723,
    "first_redis_commit_s": 26.828270639758557,
    "maximum_sampled_silence_ms": 304.404125,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 130,
    "accepted": 130,
    "sent": 50,
    "late": 0,
    "completed_fps": 4.333328998693358,
    "accepted_fps": 4.333328998693358,
    "sent_fps": 1.6666649994974454,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.574369306,
    "selected": 130,
    "infer_entries": 130,
    "infer_entry_fps": 4.251927446120317,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 5004422580,
        "mean_ms": 38.495558307692306,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          19,
          17,
          32,
          12,
          12,
          7,
          8,
          6,
          10,
          3,
          3,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 846.83915
      },
      "copy_wall": {
        "count": 130,
        "sum_ns": 7437128406,
        "mean_ms": 57.208680046153845,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          14,
          56,
          25,
          14,
          16,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 193.775706
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2381499962,
        "mean_ms": 18.319230476923078,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          9,
          80,
          40,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.781305
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 30665311160,
        "mean_ms": 235.88700892307693,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          8,
          3,
          12,
          22,
          28,
          21,
          23,
          13,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 851.710293
      },
      "selected_to_preprocess_start": {
        "count": 129,
        "sum_ns": 1815043143,
        "mean_ms": 14.07010188372093,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          31,
          19,
          25,
          27,
          19,
          5,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.750418
      },
      "preprocess_start_to_preprocess_output": {
        "count": 129,
        "sum_ns": 3178246289,
        "mean_ms": 24.637568131782945,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          1,
          1,
          4,
          28,
          54,
          34,
          5,
          0,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 766.080996
      },
      "selected_to_preprocess": {
        "count": 130,
        "sum_ns": 9138591891,
        "mean_ms": 70.2968607,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          0,
          1,
          2,
          11,
          55,
          24,
          15,
          13,
          3,
          3,
          2,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 770.024022
      },
      "preprocess_start_to_preprocess": {
        "count": 129,
        "sum_ns": 7300827962,
        "mean_ms": 56.59556559689923,
        "p95_bucket_upper_ms": 200,
        "histogram": [
          0,
          1,
          0,
          8,
          26,
          55,
          16,
          8,
          7,
          3,
          3,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 768.610735
      },
      "preprocess_output_to_infer_entry": {
        "count": 130,
        "sum_ns": 4127005546,
        "mean_ms": 31.746196507692307,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          16,
          6,
          13,
          27,
          27,
          16,
          8,
          7,
          5,
          2,
          2,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 457.604302
      },
      "preprocess_to_infer_output": {
        "count": 130,
        "sum_ns": 106128837037,
        "mean_ms": 816.3756695153846,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          8,
          97,
          24,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1447.755295
      },
      "preprocess_to_callback": {
        "count": 130,
        "sum_ns": 108152402264,
        "mean_ms": 831.9415558769231,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          7,
          97,
          26,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1459.412122
      },
      "infer_output_to_callback": {
        "count": 130,
        "sum_ns": 2023565227,
        "mean_ms": 15.565886361538462,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          19,
          37,
          35,
          32,
          5,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 101.357542
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 7634927246,
        "mean_ms": 58.73020958461539,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          14,
          55,
          26,
          15,
          16,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 194.450703
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 125418923744,
        "mean_ms": 964.7609518769231,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          70,
          57,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1722.992408
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.62981190206483,
    "first_redis_commit_s": 24.964707877952605,
    "maximum_sampled_silence_ms": 252.049772,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 169,
    "accepted": 169,
    "sent": 56,
    "late": 0,
    "completed_fps": 5.633327698301366,
    "accepted_fps": 5.633327698301366,
    "sent_fps": 1.866664799437139,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.66585642,
    "selected": 169,
    "infer_entries": 169,
    "infer_entry_fps": 5.5110151722284755,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "accepted_interval": {
        "count": 169,
        "sum_ns": 30557850755,
        "mean_ms": 180.81568494082842,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          6,
          15,
          15,
          35,
          30,
          48,
          19,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 681.003599
      },
      "completed_queue_wait": {
        "count": 169,
        "sum_ns": 513773649,
        "mean_ms": 3.0400807633136098,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          62,
          29,
          46,
          23,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 26.193039
      },
      "selected_to_preprocess_start": {
        "count": 167,
        "sum_ns": 1844937078,
        "mean_ms": 11.047527413173654,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          34,
          42,
          30,
          30,
          28,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.125084
      },
      "preprocess_start_to_preprocess_output": {
        "count": 165,
        "sum_ns": 2579575109,
        "mean_ms": 15.633788539393938,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          9,
          32,
          77,
          41,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 63.792998
      },
      "selected_to_preprocess": {
        "count": 168,
        "sum_ns": 6487856910,
        "mean_ms": 38.618195892857145,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          2,
          25,
          98,
          29,
          10,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 143.85818
      },
      "preprocess_start_to_preprocess": {
        "count": 167,
        "sum_ns": 4631192582,
        "mean_ms": 27.731692107784433,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          5,
          9,
          53,
          79,
          15,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 119.619358
      },
      "preprocess_output_to_infer_entry": {
        "count": 166,
        "sum_ns": 2005241639,
        "mean_ms": 12.079768909638554,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          35,
          6,
          20,
          36,
          47,
          14,
          6,
          1,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 116.473178
      },
      "copy_wall": {
        "count": 169,
        "sum_ns": 10955412031,
        "mean_ms": 64.82492326035504,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          72,
          40,
          26,
          22,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 294.327218
      },
      "copy_thread_cpu": {
        "count": 169,
        "sum_ns": 3581409717,
        "mean_ms": 21.19177347337278,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          79,
          89,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 164.548375
      },
      "preprocess_to_infer_output": {
        "count": 168,
        "sum_ns": 108478424851,
        "mean_ms": 645.7049098273809,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          26,
          138,
          4,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1186.242929
      },
      "preprocess_to_callback": {
        "count": 168,
        "sum_ns": 110511655280,
        "mean_ms": 657.8074719047619,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          23,
          140,
          5,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1202.380933
      },
      "infer_output_to_callback": {
        "count": 168,
        "sum_ns": 2033230429,
        "mean_ms": 12.102562077380952,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          29,
          65,
          50,
          22,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.66361
      },
      "callback_to_copied": {
        "count": 168,
        "sum_ns": 11497975244,
        "mean_ms": 68.44032883333333,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          65,
          41,
          28,
          26,
          1,
          3,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 296.404832
      },
      "selected_to_submitted": {
        "count": 168,
        "sum_ns": 128706848339,
        "mean_ms": 766.1121924940477,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          150,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1329.260373
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.44878630992025,
    "first_redis_commit_s": 24.63724011182785,
    "maximum_sampled_silence_ms": 251.63095,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 10,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 155,
    "accepted": 155,
    "sent": 52,
    "late": 0,
    "completed_fps": 5.166661498442081,
    "accepted_fps": 5.166661498442081,
    "sent_fps": 1.7333315994773433,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.411509309,
    "selected": 156,
    "infer_entries": 152,
    "infer_entry_fps": 4.998107737948311,
    "pre_infer_expired": 0,
    "diagnostic_missed": 6,
    "durations": {
      "copy_wall": {
        "count": 155,
        "sum_ns": 9317414622,
        "mean_ms": 60.1123524,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          70,
          35,
          29,
          17,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 141.315747
      },
      "copy_thread_cpu": {
        "count": 155,
        "sum_ns": 3028479731,
        "mean_ms": 19.53857890967742,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          84,
          71,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.869471
      },
      "accepted_interval": {
        "count": 155,
        "sum_ns": 30177787343,
        "mean_ms": 194.69540221290322,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          12,
          18,
          28,
          40,
          38,
          14,
          5,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 594.186639
      },
      "selected_to_preprocess_start": {
        "count": 155,
        "sum_ns": 1710964214,
        "mean_ms": 11.0384788,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          43,
          31,
          19,
          27,
          33,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.431715
      },
      "completed_queue_wait": {
        "count": 155,
        "sum_ns": 449976562,
        "mean_ms": 2.9030745935483875,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          67,
          29,
          31,
          20,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.130415
      },
      "preprocess_start_to_preprocess_output": {
        "count": 154,
        "sum_ns": 2147344271,
        "mean_ms": 13.943793967532468,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          7,
          8,
          37,
          74,
          28,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 40.999969
      },
      "selected_to_preprocess": {
        "count": 152,
        "sum_ns": 5743026894,
        "mean_ms": 37.78307167105263,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          5,
          22,
          88,
          26,
          7,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 156.406015
      },
      "preprocess_start_to_preprocess": {
        "count": 152,
        "sum_ns": 4064200933,
        "mean_ms": 26.738164032894737,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          6,
          11,
          47,
          77,
          4,
          4,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 147.616535
      },
      "preprocess_output_to_infer_entry": {
        "count": 152,
        "sum_ns": 1942563623,
        "mean_ms": 12.780023835526315,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          30,
          11,
          14,
          39,
          37,
          13,
          2,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 137.448332
      },
      "preprocess_to_infer_output": {
        "count": 153,
        "sum_ns": 107594839717,
        "mean_ms": 703.2342465163399,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          22,
          116,
          14,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1334.308232
      },
      "preprocess_to_callback": {
        "count": 153,
        "sum_ns": 109494901902,
        "mean_ms": 715.6529536078432,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          21,
          117,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1361.816549
      },
      "infer_output_to_callback": {
        "count": 155,
        "sum_ns": 1922568812,
        "mean_ms": 12.40366975483871,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          31,
          47,
          47,
          28,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 51.662576
      },
      "callback_to_copied": {
        "count": 154,
        "sum_ns": 9785201842,
        "mean_ms": 63.5402717012987,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          64,
          37,
          25,
          23,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 158.848951
      },
      "selected_to_submitted": {
        "count": 154,
        "sum_ns": 125574679088,
        "mean_ms": 815.4199940779221,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          122,
          29,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1518.076879
      }
    },
    "rejection_reasons": {},
    "first_response_s": 24.053463532123715,
    "first_redis_commit_s": 29.375223672948778,
    "maximum_sampled_silence_ms": 321.067559,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 130,
    "accepted": 130,
    "sent": 49,
    "late": 0,
    "completed_fps": 4.333328998693358,
    "accepted_fps": 4.333328998693358,
    "sent_fps": 1.6333316995074967,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.542346756,
    "selected": 130,
    "infer_entries": 130,
    "infer_entry_fps": 4.256385438832125,
    "pre_infer_expired": 0,
    "diagnostic_missed": 1,
    "durations": {
      "copy_wall": {
        "count": 130,
        "sum_ns": 8024360679,
        "mean_ms": 61.725851376923075,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          3,
          15,
          49,
          26,
          22,
          12,
          1,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 560.178115
      },
      "copy_thread_cpu": {
        "count": 130,
        "sum_ns": 2775653156,
        "mean_ms": 21.351178123076924,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          13,
          91,
          24,
          0,
          0,
          0,
          0,
          0,
          2,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 378.047319
      },
      "accepted_interval": {
        "count": 130,
        "sum_ns": 30496441472,
        "mean_ms": 234.58801132307693,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          3,
          4,
          12,
          19,
          29,
          28,
          24,
          10,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 912.5534
      },
      "completed_queue_wait": {
        "count": 130,
        "sum_ns": 3377601747,
        "mean_ms": 25.9815519,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          21,
          20,
          26,
          15,
          10,
          13,
          7,
          10,
          4,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 241.760087
      },
      "selected_to_preprocess_start": {
        "count": 130,
        "sum_ns": 1507820239,
        "mean_ms": 11.598617223076923,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          32,
          28,
          14,
          35,
          20,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 86.026173
      },
      "preprocess_start_to_preprocess_output": {
        "count": 129,
        "sum_ns": 2387019783,
        "mean_ms": 18.504029325581396,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          12,
          24,
          49,
          38,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 68.426208
      },
      "selected_to_preprocess": {
        "count": 130,
        "sum_ns": 6370740899,
        "mean_ms": 49.005699223076924,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          4,
          17,
          59,
          26,
          15,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 182.031904
      },
      "preprocess_start_to_preprocess": {
        "count": 130,
        "sum_ns": 4862920660,
        "mean_ms": 37.407082,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          2,
          1,
          7,
          36,
          54,
          16,
          10,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 180.125279
      },
      "preprocess_output_to_infer_entry": {
        "count": 129,
        "sum_ns": 2458885191,
        "mean_ms": 19.06112551162791,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          15,
          10,
          13,
          23,
          26,
          30,
          6,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.13265
      },
      "preprocess_to_infer_output": {
        "count": 130,
        "sum_ns": 106313549918,
        "mean_ms": 817.7965378307691,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          106,
          18,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1371.245452
      },
      "preprocess_to_callback": {
        "count": 130,
        "sum_ns": 108016575548,
        "mean_ms": 830.8967349846154,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          103,
          22,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1378.725726
      },
      "infer_output_to_callback": {
        "count": 130,
        "sum_ns": 1703025630,
        "mean_ms": 13.100197153846153,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          26,
          38,
          39,
          25,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 53.590304
      },
      "callback_to_copied": {
        "count": 130,
        "sum_ns": 8321180832,
        "mean_ms": 64.00908332307692,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          3,
          13,
          47,
          26,
          25,
          13,
          1,
          0,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 563.435898
      },
      "selected_to_submitted": {
        "count": 130,
        "sum_ns": 123489104256,
        "mean_ms": 949.9161865846154,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          81,
          48,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1593.581823
      }
    },
    "rejection_reasons": {},
    "first_response_s": 28.138724002987146,
    "first_redis_commit_s": 28.43517256109044,
    "maximum_sampled_silence_ms": 318.650451,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 170,
    "accepted": 170,
    "sent": 55,
    "late": 0,
    "completed_fps": 5.666660998291315,
    "accepted_fps": 5.666660998291315,
    "sent_fps": 1.83333149944719,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.280683007,
    "selected": 170,
    "infer_entries": 167,
    "infer_entry_fps": 5.515067145658324,
    "pre_infer_expired": 0,
    "diagnostic_missed": 8,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 168,
        "sum_ns": 1389666487,
        "mean_ms": 8.271824327380953,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          45,
          49,
          27,
          28,
          18,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.173378
      },
      "preprocess_start_to_preprocess_output": {
        "count": 166,
        "sum_ns": 2287708271,
        "mean_ms": 13.781375126506026,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          4,
          18,
          43,
          62,
          36,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.553316
      },
      "selected_to_preprocess": {
        "count": 165,
        "sum_ns": 5721320623,
        "mean_ms": 34.674670442424244,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          9,
          39,
          89,
          15,
          8,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 191.795912
      },
      "preprocess_start_to_preprocess": {
        "count": 165,
        "sum_ns": 4388473210,
        "mean_ms": 26.59680733333333,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          7,
          25,
          54,
          64,
          6,
          6,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 188.426068
      },
      "preprocess_output_to_infer_entry": {
        "count": 164,
        "sum_ns": 2072254829,
        "mean_ms": 12.63570017682927,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          42,
          12,
          17,
          34,
          33,
          17,
          5,
          1,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 186.056927
      },
      "copy_wall": {
        "count": 170,
        "sum_ns": 9879007080,
        "mean_ms": 58.11180635294118,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          76,
          43,
          29,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 187.021319
      },
      "copy_thread_cpu": {
        "count": 170,
        "sum_ns": 3192015207,
        "mean_ms": 18.77656004117647,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          108,
          62,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.554989
      },
      "accepted_interval": {
        "count": 170,
        "sum_ns": 30355734736,
        "mean_ms": 178.56314550588235,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          5,
          16,
          24,
          34,
          36,
          29,
          26,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 488.800293
      },
      "completed_queue_wait": {
        "count": 170,
        "sum_ns": 551281716,
        "mean_ms": 3.2428336235294117,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          52,
          33,
          48,
          28,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 20.735725
      },
      "preprocess_to_infer_output": {
        "count": 166,
        "sum_ns": 106080926406,
        "mean_ms": 639.0417253373494,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          38,
          126,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1094.817806
      },
      "preprocess_to_callback": {
        "count": 166,
        "sum_ns": 107739485931,
        "mean_ms": 649.0330477771084,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          37,
          127,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1100.079561
      },
      "infer_output_to_callback": {
        "count": 168,
        "sum_ns": 1682198068,
        "mean_ms": 10.013083738095238,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          4,
          35,
          68,
          49,
          11,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 52.335355
      },
      "callback_to_copied": {
        "count": 167,
        "sum_ns": 10150773037,
        "mean_ms": 60.78307207784432,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          71,
          42,
          31,
          15,
          2,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 187.685981
      },
      "selected_to_submitted": {
        "count": 168,
        "sum_ns": 125581070306,
        "mean_ms": 747.5063708690476,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          13,
          140,
          15,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1324.824701
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.699757964815944,
    "first_redis_commit_s": 27.95031144982204,
    "maximum_sampled_silence_ms": 303.591373,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 11,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 152,
    "accepted": 152,
    "sent": 48,
    "late": 0,
    "completed_fps": 5.066661598472234,
    "accepted_fps": 5.066661598472234,
    "sent_fps": 1.5999983995175477,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 29.980761614,
    "selected": 152,
    "infer_entries": 151,
    "infer_entry_fps": 5.036563178217865,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "completed_queue_wait": {
        "count": 152,
        "sum_ns": 387910112,
        "mean_ms": 2.552040210526316,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          59,
          33,
          43,
          11,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 24.002699
      },
      "copy_wall": {
        "count": 152,
        "sum_ns": 8290533246,
        "mean_ms": 54.542981881578946,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          3,
          81,
          37,
          13,
          17,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 180.628073
      },
      "copy_thread_cpu": {
        "count": 152,
        "sum_ns": 2922683747,
        "mean_ms": 19.22818254605263,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          88,
          64,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 32.296839
      },
      "accepted_interval": {
        "count": 152,
        "sum_ns": 29948153101,
        "mean_ms": 197.02732303289474,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          7,
          15,
          10,
          29,
          27,
          39,
          21,
          4,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 633.906323
      },
      "selected_to_preprocess_start": {
        "count": 150,
        "sum_ns": 1233396220,
        "mean_ms": 8.222641466666667,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          39,
          36,
          27,
          34,
          13,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 46.104106
      },
      "preprocess_start_to_preprocess_output": {
        "count": 150,
        "sum_ns": 1942311266,
        "mean_ms": 12.948741773333333,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          3,
          2,
          12,
          46,
          62,
          25,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.230882
      },
      "selected_to_preprocess": {
        "count": 151,
        "sum_ns": 4873164312,
        "mean_ms": 32.272611337748344,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          1,
          7,
          35,
          87,
          15,
          4,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 231.889712
      },
      "preprocess_start_to_preprocess": {
        "count": 149,
        "sum_ns": 3525101096,
        "mean_ms": 23.658396617449664,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          5,
          11,
          51,
          74,
          2,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 226.314766
      },
      "preprocess_output_to_infer_entry": {
        "count": 151,
        "sum_ns": 1604123307,
        "mean_ms": 10.623333158940397,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          35,
          9,
          22,
          33,
          34,
          13,
          1,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 214.881696
      },
      "preprocess_to_infer_output": {
        "count": 151,
        "sum_ns": 108073201272,
        "mean_ms": 715.7165647152318,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          127,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1352.820912
      },
      "preprocess_to_callback": {
        "count": 151,
        "sum_ns": 109666691796,
        "mean_ms": 726.2694820927153,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          14,
          127,
          10,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1362.734873
      },
      "infer_output_to_callback": {
        "count": 152,
        "sum_ns": 1599094519,
        "mean_ms": 10.52035867763158,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          40,
          57,
          40,
          13,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 54.782522
      },
      "callback_to_copied": {
        "count": 152,
        "sum_ns": 8663720649,
        "mean_ms": 56.99816216447368,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          2,
          76,
          40,
          14,
          19,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 181.599397
      },
      "selected_to_submitted": {
        "count": 152,
        "sum_ns": 124229436429,
        "mean_ms": 817.298923875,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          123,
          23,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1458.268128
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.559390062931925,
    "first_redis_commit_s": 30.789499973878264,
    "maximum_sampled_silence_ms": 310.41479,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "pose",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 129,
    "accepted": 129,
    "sent": 50,
    "late": 0,
    "completed_fps": 4.29999569870341,
    "accepted_fps": 4.29999569870341,
    "sent_fps": 1.6666649994974454,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.103291236,
    "selected": 130,
    "infer_entries": 129,
    "infer_entry_fps": 4.285245722425565,
    "pre_infer_expired": 0,
    "diagnostic_missed": 4,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 126,
        "sum_ns": 1498982900,
        "mean_ms": 11.896689682539682,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          22,
          33,
          23,
          25,
          19,
          4,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 72.423706
      },
      "copy_wall": {
        "count": 129,
        "sum_ns": 6194012803,
        "mean_ms": 48.01560312403101,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          24,
          61,
          25,
          6,
          5,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 615.538615
      },
      "copy_thread_cpu": {
        "count": 129,
        "sum_ns": 2047152171,
        "mean_ms": 15.869396674418605,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          31,
          86,
          11,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 380.314945
      },
      "accepted_interval": {
        "count": 129,
        "sum_ns": 29982228332,
        "mean_ms": 232.42037466666665,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          6,
          17,
          18,
          22,
          29,
          25,
          9,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 855.24523
      },
      "preprocess_start_to_preprocess_output": {
        "count": 126,
        "sum_ns": 2099416472,
        "mean_ms": 16.66203549206349,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          6,
          7,
          29,
          47,
          33,
          3,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 78.648803
      },
      "completed_queue_wait": {
        "count": 129,
        "sum_ns": 2492251147,
        "mean_ms": 19.319776333333333,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          19,
          22,
          33,
          18,
          17,
          6,
          3,
          3,
          3,
          4,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 257.461829
      },
      "selected_to_preprocess": {
        "count": 126,
        "sum_ns": 5814199823,
        "mean_ms": 46.144443039682535,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          1,
          3,
          17,
          65,
          25,
          11,
          2,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 243.675532
      },
      "preprocess_start_to_preprocess": {
        "count": 126,
        "sum_ns": 4300366028,
        "mean_ms": 34.12988911111111,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          4,
          8,
          34,
          58,
          13,
          6,
          1,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 235.915394
      },
      "preprocess_output_to_infer_entry": {
        "count": 126,
        "sum_ns": 2198340098,
        "mean_ms": 17.447143634920636,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          13,
          9,
          28,
          21,
          28,
          17,
          5,
          2,
          1,
          0,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 224.996525
      },
      "preprocess_to_infer_output": {
        "count": 126,
        "sum_ns": 102534155937,
        "mean_ms": 813.7631423571428,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          3,
          105,
          18,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1316.871208
      },
      "preprocess_to_callback": {
        "count": 126,
        "sum_ns": 104119294033,
        "mean_ms": 826.3436034365079,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          2,
          105,
          19,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1327.065681
      },
      "infer_output_to_callback": {
        "count": 127,
        "sum_ns": 1610278338,
        "mean_ms": 12.679356992125983,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          24,
          39,
          39,
          22,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 55.540178
      },
      "callback_to_copied": {
        "count": 126,
        "sum_ns": 6306386786,
        "mean_ms": 50.05068877777778,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          5,
          17,
          64,
          25,
          7,
          5,
          1,
          0,
          0,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 616.228731
      },
      "selected_to_submitted": {
        "count": 126,
        "sum_ns": 116705051946,
        "mean_ms": 926.230571,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          85,
          40,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1559.399622
      }
    },
    "rejection_reasons": {},
    "first_response_s": 28.142462809104472,
    "first_redis_commit_s": 28.555929981172085,
    "maximum_sampled_silence_ms": 579.339097,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "fire",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 166,
    "accepted": 166,
    "sent": 56,
    "late": 0,
    "completed_fps": 5.533327798331519,
    "accepted_fps": 5.533327798331519,
    "sent_fps": 1.866664799437139,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.11386633,
    "selected": 166,
    "infer_entries": 166,
    "infer_entry_fps": 5.51241073400886,
    "pre_infer_expired": 0,
    "diagnostic_missed": 0,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 166,
        "sum_ns": 2134048761,
        "mean_ms": 12.855715427710845,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          28,
          38,
          25,
          39,
          33,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 72.144387
      },
      "copy_wall": {
        "count": 166,
        "sum_ns": 11066008224,
        "mean_ms": 66.66270014457831,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          60,
          53,
          25,
          23,
          3,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 221.363997
      },
      "copy_thread_cpu": {
        "count": 166,
        "sum_ns": 3330404861,
        "mean_ms": 20.06267988554217,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          81,
          85,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.940168
      },
      "accepted_interval": {
        "count": 166,
        "sum_ns": 30209439273,
        "mean_ms": 181.98457393373496,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          3,
          12,
          21,
          29,
          41,
          42,
          17,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 514.55851
      },
      "preprocess_start_to_preprocess_output": {
        "count": 166,
        "sum_ns": 2518154513,
        "mean_ms": 15.1696055,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          2,
          10,
          39,
          79,
          33,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 50.962601
      },
      "completed_queue_wait": {
        "count": 166,
        "sum_ns": 674084589,
        "mean_ms": 4.060750536144578,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          50,
          37,
          43,
          20,
          10,
          5,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 59.979992
      },
      "selected_to_preprocess": {
        "count": 166,
        "sum_ns": 7032087644,
        "mean_ms": 42.36197375903615,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          3,
          5,
          21,
          86,
          36,
          7,
          7,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 177.342906
      },
      "preprocess_start_to_preprocess": {
        "count": 166,
        "sum_ns": 4898038883,
        "mean_ms": 29.5062583313253,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          2,
          3,
          16,
          42,
          80,
          17,
          1,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 165.773462
      },
      "preprocess_output_to_infer_entry": {
        "count": 166,
        "sum_ns": 2379884370,
        "mean_ms": 14.336652831325303,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          30,
          8,
          18,
          36,
          47,
          20,
          2,
          2,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 156.039009
      },
      "preprocess_to_infer_output": {
        "count": 165,
        "sum_ns": 105763093107,
        "mean_ms": 640.9884430727274,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          1,
          35,
          126,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1028.620637
      },
      "preprocess_to_callback": {
        "count": 165,
        "sum_ns": 107761340305,
        "mean_ms": 653.0990321515151,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          34,
          128,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1043.634593
      },
      "infer_output_to_callback": {
        "count": 165,
        "sum_ns": 1998247198,
        "mean_ms": 12.110589078787878,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          22,
          54,
          63,
          24,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 45.400203
      },
      "callback_to_copied": {
        "count": 165,
        "sum_ns": 11594118861,
        "mean_ms": 70.26738703636363,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          1,
          52,
          55,
          26,
          27,
          2,
          2,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 222.336467
      },
      "selected_to_submitted": {
        "count": 165,
        "sum_ns": 126626859390,
        "mean_ms": 767.4355114545455,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          141,
          19,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1285.402527
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.594960568938404,
    "first_redis_commit_s": 27.834589728154242,
    "maximum_sampled_silence_ms": 235.571966,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 12,
    "provider": "crowd",
    "cohort": "new",
    "seconds": 30.000030009076,
    "completed": 154,
    "accepted": 154,
    "sent": 46,
    "late": 0,
    "completed_fps": 5.133328198452132,
    "accepted_fps": 5.133328198452132,
    "sent_fps": 1.5333317995376499,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.301454824,
    "selected": 154,
    "infer_entries": 154,
    "infer_entry_fps": 5.08226423102384,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 153,
        "sum_ns": 1256788439,
        "mean_ms": 8.214303522875817,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          36,
          44,
          27,
          31,
          15,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 37.27894
      },
      "preprocess_start_to_preprocess_output": {
        "count": 152,
        "sum_ns": 2098853858,
        "mean_ms": 13.808249065789473,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          10,
          42,
          71,
          25,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 71.888384
      },
      "copy_wall": {
        "count": 154,
        "sum_ns": 8301924162,
        "mean_ms": 53.908598454545455,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          72,
          47,
          19,
          9,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.252641
      },
      "copy_thread_cpu": {
        "count": 154,
        "sum_ns": 2880620685,
        "mean_ms": 18.70532912337662,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          94,
          60,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 30.249721
      },
      "accepted_interval": {
        "count": 154,
        "sum_ns": 30379989157,
        "mean_ms": 197.27265686363637,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          12,
          21,
          25,
          26,
          40,
          24,
          2,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 560.295873
      },
      "completed_queue_wait": {
        "count": 154,
        "sum_ns": 339157025,
        "mean_ms": 2.2023183441558443,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          60,
          39,
          40,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 19.47772
      },
      "selected_to_preprocess": {
        "count": 154,
        "sum_ns": 5494487027,
        "mean_ms": 35.67848718831169,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          2,
          8,
          27,
          89,
          21,
          2,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 170.367324
      },
      "preprocess_start_to_preprocess": {
        "count": 153,
        "sum_ns": 4184093203,
        "mean_ms": 27.34701439869281,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          1,
          7,
          17,
          49,
          65,
          9,
          0,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 167.790082
      },
      "preprocess_output_to_infer_entry": {
        "count": 153,
        "sum_ns": 2078636057,
        "mean_ms": 13.585856581699348,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          32,
          14,
          16,
          38,
          29,
          17,
          2,
          0,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 154.654024
      },
      "preprocess_to_infer_output": {
        "count": 151,
        "sum_ns": 107412552774,
        "mean_ms": 711.3414090993377,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          19,
          123,
          9,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1288.566132
      },
      "preprocess_to_callback": {
        "count": 151,
        "sum_ns": 109364302773,
        "mean_ms": 724.2669057814569,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          15,
          127,
          9,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1302.212037
      },
      "infer_output_to_callback": {
        "count": 151,
        "sum_ns": 1951749999,
        "mean_ms": 12.925496682119206,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          2,
          26,
          38,
          60,
          24,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 84.24852
      },
      "callback_to_copied": {
        "count": 151,
        "sum_ns": 8531542546,
        "mean_ms": 56.5002817615894,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          4,
          67,
          48,
          22,
          10,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 133.896573
      },
      "selected_to_submitted": {
        "count": 151,
        "sum_ns": 123533424042,
        "mean_ms": 818.1021459735099,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          124,
          21,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1443.639048
      }
    },
    "rejection_reasons": {},
    "first_response_s": 27.813052386045456,
    "first_redis_commit_s": 31.468997600954026,
    "maximum_sampled_silence_ms": 189.885382,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "pose",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 127,
    "accepted": 127,
    "sent": 50,
    "late": 0,
    "completed_fps": 4.2333290987235115,
    "accepted_fps": 4.2333290987235115,
    "sent_fps": 1.6666649994974454,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.420893809,
    "selected": 127,
    "infer_entries": 127,
    "infer_entry_fps": 4.174762280075648,
    "pre_infer_expired": 0,
    "diagnostic_missed": 5,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 124,
        "sum_ns": 3529826122,
        "mean_ms": 28.466339693548388,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          18,
          23,
          24,
          15,
          28,
          11,
          3,
          0,
          1,
          0,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1077.310736
      },
      "preprocess_start_to_preprocess_output": {
        "count": 124,
        "sum_ns": 2863161446,
        "mean_ms": 23.090011661290323,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          2,
          4,
          20,
          55,
          31,
          8,
          1,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 187.979981
      },
      "selected_to_preprocess": {
        "count": 125,
        "sum_ns": 9500039124,
        "mean_ms": 76.000312992,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          1,
          10,
          50,
          31,
          13,
          15,
          2,
          1,
          0,
          0,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1093.285652
      },
      "preprocess_start_to_preprocess": {
        "count": 124,
        "sum_ns": 5901460170,
        "mean_ms": 47.592420725806456,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          1,
          5,
          26,
          60,
          16,
          7,
          7,
          1,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 994.960741
      },
      "preprocess_output_to_infer_entry": {
        "count": 125,
        "sum_ns": 3059540577,
        "mean_ms": 24.476324616,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          9,
          7,
          17,
          23,
          37,
          20,
          6,
          4,
          1,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 943.843334
      },
      "copy_wall": {
        "count": 127,
        "sum_ns": 9159394381,
        "mean_ms": 72.1212155984252,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          7,
          52,
          25,
          22,
          18,
          0,
          2,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1074.073629
      },
      "copy_thread_cpu": {
        "count": 127,
        "sum_ns": 3617539873,
        "mean_ms": 28.484565929133858,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          1,
          63,
          59,
          3,
          0,
          0,
          0,
          0,
          0,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 699.722439
      },
      "accepted_interval": {
        "count": 127,
        "sum_ns": 30359583540,
        "mean_ms": 239.0518388976378,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          7,
          9,
          19,
          28,
          26,
          28,
          4,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1269.757308
      },
      "completed_queue_wait": {
        "count": 126,
        "sum_ns": 6821282802,
        "mean_ms": 54.137165095238096,
        "p95_bucket_upper_ms": 300,
        "histogram": [
          24,
          20,
          22,
          13,
          6,
          4,
          7,
          7,
          5,
          8,
          8,
          1,
          1,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 934.241823
      },
      "preprocess_to_infer_output": {
        "count": 125,
        "sum_ns": 107987651765,
        "mean_ms": 863.90121412,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          84,
          35,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1878.859635
      },
      "preprocess_to_callback": {
        "count": 125,
        "sum_ns": 110004666176,
        "mean_ms": 880.037329408,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          85,
          35,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1893.74582
      },
      "infer_output_to_callback": {
        "count": 125,
        "sum_ns": 2017014411,
        "mean_ms": 16.136115288,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          1,
          14,
          25,
          53,
          29,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 91.347056
      },
      "callback_to_copied": {
        "count": 125,
        "sum_ns": 9376307888,
        "mean_ms": 75.010463104,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          45,
          29,
          23,
          18,
          1,
          2,
          0,
          0,
          1,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1074.659679
      },
      "selected_to_submitted": {
        "count": 125,
        "sum_ns": 129338880838,
        "mean_ms": 1034.711046704,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          74,
          48,
          3,
          0
        ],
        "diagnostic_window_max_ms": 2116.824302
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.32267033495009,
    "first_redis_commit_s": 22.42273849574849,
    "maximum_sampled_silence_ms": 1232.016107,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "fire",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 172,
    "accepted": 172,
    "sent": 55,
    "late": 0,
    "completed_fps": 5.733327598271212,
    "accepted_fps": 5.733327598271212,
    "sent_fps": 1.83333149944719,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.609387875,
    "selected": 173,
    "infer_entries": 172,
    "infer_entry_fps": 5.619191102494401,
    "pre_infer_expired": 0,
    "diagnostic_missed": 3,
    "durations": {
      "selected_to_preprocess_start": {
        "count": 172,
        "sum_ns": 2208177424,
        "mean_ms": 12.838240837209304,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          23,
          34,
          35,
          49,
          28,
          2,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 76.051106
      },
      "preprocess_start_to_preprocess_output": {
        "count": 170,
        "sum_ns": 2813577957,
        "mean_ms": 16.550458570588233,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          1,
          4,
          41,
          77,
          45,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 64.057712
      },
      "selected_to_preprocess": {
        "count": 172,
        "sum_ns": 7363619697,
        "mean_ms": 42.811742424418604,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          1,
          13,
          107,
          36,
          13,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 113.330471
      },
      "preprocess_start_to_preprocess": {
        "count": 172,
        "sum_ns": 5155442273,
        "mean_ms": 29.973501587209302,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          4,
          4,
          46,
          97,
          16,
          4,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 105.079382
      },
      "preprocess_output_to_infer_entry": {
        "count": 170,
        "sum_ns": 2280396446,
        "mean_ms": 13.414096741176472,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          22,
          10,
          9,
          49,
          50,
          21,
          7,
          2,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 93.19892
      },
      "copy_wall": {
        "count": 172,
        "sum_ns": 10918291197,
        "mean_ms": 63.478437191860465,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          65,
          50,
          23,
          25,
          3,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 166.596503
      },
      "copy_thread_cpu": {
        "count": 172,
        "sum_ns": 3503821625,
        "mean_ms": 20.371055959302325,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          82,
          90,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 35.9453
      },
      "accepted_interval": {
        "count": 172,
        "sum_ns": 30482818434,
        "mean_ms": 177.22568856976744,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          6,
          12,
          23,
          39,
          30,
          43,
          19,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 581.407932
      },
      "completed_queue_wait": {
        "count": 172,
        "sum_ns": 560066038,
        "mean_ms": 3.256197895348837,
        "p95_bucket_upper_ms": 20,
        "histogram": [
          56,
          37,
          42,
          26,
          10,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 22.576809
      },
      "preprocess_to_infer_output": {
        "count": 173,
        "sum_ns": 111937332079,
        "mean_ms": 647.0366016127168,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          30,
          141,
          2,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1194.72959
      },
      "preprocess_to_callback": {
        "count": 172,
        "sum_ns": 113964792058,
        "mean_ms": 662.5860003372094,
        "p95_bucket_upper_ms": 1000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          26,
          143,
          3,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1240.024467
      },
      "infer_output_to_callback": {
        "count": 172,
        "sum_ns": 2495911851,
        "mean_ms": 14.511115412790698,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          35,
          42,
          52,
          43,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 47.103836
      },
      "callback_to_copied": {
        "count": 171,
        "sum_ns": 11487371351,
        "mean_ms": 67.17761023976608,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          5,
          57,
          52,
          25,
          28,
          4,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 176.989789
      },
      "selected_to_submitted": {
        "count": 172,
        "sum_ns": 132945994452,
        "mean_ms": 772.9418282093023,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          5,
          155,
          12,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1390.964338
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.058295923750848,
    "first_redis_commit_s": 22.104469902813435,
    "maximum_sampled_silence_ms": 344.18056,
    "redis_commit_p95_bucket_upper_ms": 2000
  },
  {
    "level": 8,
    "phase_status": "PASS",
    "camera": 3,
    "provider": "crowd",
    "cohort": "existing",
    "seconds": 30.000030009076,
    "completed": 158,
    "accepted": 158,
    "sent": 46,
    "late": 0,
    "completed_fps": 5.2666613984119275,
    "accepted_fps": 5.2666613984119275,
    "sent_fps": 1.5333317995376499,
    "rejected_pct": 0.0,
    "diagnostic_complete": true,
    "diagnostic_scope": "sample_difference",
    "diagnostic_seconds": 30.392162034,
    "selected": 158,
    "infer_entries": 157,
    "infer_entry_fps": 5.165805572646086,
    "pre_infer_expired": 0,
    "diagnostic_missed": 2,
    "durations": {
      "accepted_interval": {
        "count": 158,
        "sum_ns": 30429488294,
        "mean_ms": 192.59169806329112,
        "p95_bucket_upper_ms": 500,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          4,
          15,
          15,
          40,
          25,
          39,
          13,
          7,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 703.282137
      },
      "completed_queue_wait": {
        "count": 158,
        "sum_ns": 492052060,
        "mean_ms": 3.1142535443037977,
        "p95_bucket_upper_ms": 10,
        "histogram": [
          61,
          32,
          38,
          20,
          4,
          3,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 36.075961
      },
      "selected_to_preprocess_start": {
        "count": 157,
        "sum_ns": 1818368282,
        "mean_ms": 11.581963579617833,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          29,
          32,
          25,
          43,
          27,
          0,
          0,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 112.955646
      },
      "preprocess_start_to_preprocess_output": {
        "count": 157,
        "sum_ns": 2252259413,
        "mean_ms": 14.345601356687897,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          1,
          1,
          2,
          40,
          85,
          28,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 41.189083
      },
      "copy_wall": {
        "count": 158,
        "sum_ns": 8337346165,
        "mean_ms": 52.76801370253164,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          13,
          79,
          38,
          17,
          10,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 230.249289
      },
      "copy_thread_cpu": {
        "count": 158,
        "sum_ns": 3126928972,
        "mean_ms": 19.79068969620253,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          0,
          0,
          83,
          75,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 34.759625
      },
      "selected_to_preprocess": {
        "count": 156,
        "sum_ns": 6511252324,
        "mean_ms": 41.738796948717955,
        "p95_bucket_upper_ms": 100,
        "histogram": [
          0,
          0,
          0,
          2,
          21,
          100,
          22,
          4,
          5,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 256.933822
      },
      "preprocess_start_to_preprocess": {
        "count": 156,
        "sum_ns": 4718640471,
        "mean_ms": 30.247695326923075,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          0,
          0,
          0,
          8,
          53,
          80,
          8,
          1,
          4,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 255.064495
      },
      "preprocess_output_to_infer_entry": {
        "count": 156,
        "sum_ns": 2507570141,
        "mean_ms": 16.07416757051282,
        "p95_bucket_upper_ms": 75,
        "histogram": [
          23,
          7,
          11,
          44,
          44,
          16,
          4,
          3,
          2,
          1,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 233.580936
      },
      "infer_output_to_callback": {
        "count": 157,
        "sum_ns": 2159649526,
        "mean_ms": 13.755729464968152,
        "p95_bucket_upper_ms": 50,
        "histogram": [
          0,
          0,
          23,
          46,
          56,
          31,
          1,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 57.683183
      },
      "callback_to_copied": {
        "count": 157,
        "sum_ns": 8884502613,
        "mean_ms": 56.589188617834395,
        "p95_bucket_upper_ms": 150,
        "histogram": [
          0,
          0,
          0,
          0,
          6,
          77,
          42,
          19,
          12,
          0,
          1,
          0,
          0,
          0,
          0,
          0
        ],
        "diagnostic_window_max_ms": 258.51619
      },
      "selected_to_submitted": {
        "count": 157,
        "sum_ns": 129095044932,
        "mean_ms": 822.261432687898,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          6,
          123,
          28,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1516.429371
      },
      "preprocess_to_infer_output": {
        "count": 156,
        "sum_ns": 110690080755,
        "mean_ms": 709.5517997115385,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          21,
          122,
          13,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1390.812907
      },
      "preprocess_to_callback": {
        "count": 156,
        "sum_ns": 112835650995,
        "mean_ms": 723.3054550961539,
        "p95_bucket_upper_ms": 2000,
        "histogram": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          16,
          126,
          14,
          0,
          0
        ],
        "diagnostic_window_max_ms": 1405.583005
      }
    },
    "rejection_reasons": {},
    "first_response_s": 22.125111697707325,
    "first_redis_commit_s": 24.006831201724708,
    "maximum_sampled_silence_ms": 518.452941,
    "redis_commit_p95_bucket_upper_ms": 2000
  }
]