{
  "verified": "2026-10-23",
  "githubCommit": "094fdecfee669cb15dd30c1fc663bfad55394f14",
  "datasetCommit": "40285b3a4a91e3aeefd0c8f5544f5f8e94a4c3ac",
  "source": "https://github.com/lakeday-org/perch/tree/094fdecfee669cb15dd30c1fc663bfad55394f14/.benchmarks",
  "dataset": "https://huggingface.co/datasets/perchscan/benchmark-results/tree/40285b3a4a91e3aeefd0c8f5544f5f8e94a4c3ac",
  "greptile": [
    {
      "config": "oai2q",
      "label": "perch#418, OpenAI Decisions",
      "cutoff": 0.6,
      "prs": 50,
      "missing": [],
      "known": 38,
      "known_of": 50,
      "known_chance": 20.2,
      "caught": 42,
      "of": 47,
      "chance": 32,
      "golden": 116,
      "golden_of": 140,
      "flags": 490,
      "read": 1568,
      "flag_rate": 0.313,
      "median_flags": 7,
      "cost": 0.335
    },
    {
      "config": "oai2q",
      "label": "perch#418, OpenAI Decisions",
      "cutoff": 0.7,
      "prs": 50,
      "missing": [],
      "known": 33,
      "known_of": 50,
      "known_chance": 16.3,
      "caught": 39,
      "of": 47,
      "chance": 27.3,
      "golden": 105,
      "golden_of": 140,
      "flags": 383,
      "read": 1568,
      "flag_rate": 0.244,
      "median_flags": 6,
      "cost": 0.335
    },
    {
      "config": "oai2q",
      "label": "perch#418, OpenAI Decisions",
      "cutoff": 0.8,
      "prs": 50,
      "missing": [],
      "known": 28,
      "known_of": 50,
      "known_chance": 11.8,
      "caught": 39,
      "of": 47,
      "chance": 21.5,
      "golden": 86,
      "golden_of": 140,
      "flags": 264,
      "read": 1568,
      "flag_rate": 0.168,
      "median_flags": 4,
      "cost": 0.335
    },
    {
      "config": "jev048",
      "label": "perch 0.4.8, Jev",
      "cutoff": 0.6,
      "prs": 50,
      "missing": [],
      "known": 22,
      "known_of": 50,
      "known_chance": 8.2,
      "caught": 27,
      "of": 47,
      "chance": 15.2,
      "golden": 65,
      "golden_of": 140,
      "flags": 179,
      "read": 1568,
      "flag_rate": 0.114,
      "median_flags": 2,
      "cost": 0.149
    },
    {
      "config": "jev048",
      "label": "perch 0.4.8, Jev",
      "cutoff": 0.7,
      "prs": 50,
      "missing": [],
      "known": 12,
      "known_of": 50,
      "known_chance": 4,
      "caught": 19,
      "of": 47,
      "chance": 8.8,
      "golden": 37,
      "golden_of": 140,
      "flags": 75,
      "read": 1568,
      "flag_rate": 0.048,
      "median_flags": 0,
      "cost": 0.149
    },
    {
      "config": "jev048",
      "label": "perch 0.4.8, Jev",
      "cutoff": 0.8,
      "prs": 50,
      "missing": [],
      "known": 6,
      "known_of": 50,
      "known_chance": 0.7,
      "caught": 8,
      "of": 47,
      "chance": 2.2,
      "golden": 13,
      "golden_of": 140,
      "flags": 18,
      "read": 1568,
      "flag_rate": 0.011,
      "median_flags": 0,
      "cost": 0.149
    }
  ],
  "reviewbench": [
    {
      "model": "OpenAI Decisions",
      "prs": 171,
      "targets": 16168,
      "buggy": 1506,
      "modelCostUsd": 4.1569033,
      "cutoffs": [
        {
          "cutoff": 0.6,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 2094,
          "flag_rate": 0.12951509153884216,
          "precision": 0.28796561604584525,
          "recall": 0.40039840637450197,
          "f1": 0.335,
          "correct": 603,
          "false": 1491,
          "missed": 903
        },
        {
          "cutoff": 0.7,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 1364,
          "flag_rate": 0.08436417615042058,
          "precision": 0.3519061583577713,
          "recall": 0.3187250996015936,
          "f1": 0.3344947735191638,
          "correct": 480,
          "false": 884,
          "missed": 1026
        },
        {
          "cutoff": 0.8,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 836,
          "flag_rate": 0.051707075705096486,
          "precision": 0.41746411483253587,
          "recall": 0.23173970783532535,
          "f1": 0.29803586678052946,
          "correct": 349,
          "false": 487,
          "missed": 1157
        }
      ]
    },
    {
      "model": "Jev",
      "prs": 171,
      "targets": 16168,
      "buggy": 1506,
      "modelCostUsd": 1.798197534,
      "cutoffs": [
        {
          "cutoff": 0.6,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 260,
          "flag_rate": 0.016081147946561108,
          "precision": 0.4576923076923077,
          "recall": 0.07901726427622842,
          "f1": 0.1347678369195923,
          "correct": 119,
          "false": 141,
          "missed": 1387
        },
        {
          "cutoff": 0.7,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 81,
          "flag_rate": 0.0050098960910440376,
          "precision": 0.5061728395061729,
          "recall": 0.027224435590969456,
          "f1": 0.0516698172652804,
          "correct": 41,
          "false": 40,
          "missed": 1465
        },
        {
          "cutoff": 0.8,
          "targets": 16168,
          "buggy": 1506,
          "flagged": 7,
          "flag_rate": 0.0004329539831766452,
          "precision": 0.8571428571428571,
          "recall": 0.00398406374501992,
          "f1": 0.007931262392597489,
          "correct": 6,
          "false": 1,
          "missed": 1500
        }
      ]
    }
  ],
  "scoring": "Perch is credited for flagging the function containing a labelled bug. Published competitor scores credit judged review comments. These are different scoring units; the figures are not an identical-protocol ranking.",
  "costBasis": "Measured input-token model cost for each saved run, not the hosted Perch price or a guaranteed cost for another pull request."
}