{
  "id": "order-service-behavior-2026-10-14",
  "captured_date": "2026-10-14",
  "publisher": "Perch",
  "source": {
    "source_commit": "4af67af46d76ac99398d61c7ad0a45441760d4dc",
    "base_pr": "https://github.com/lakeday-org/perch/pull/370",
    "shared_instructions_sha256": "86aff672df11834e33d9ac841b9c0dba57d55be1c2e01eacac6f1d75fc4e1e07",
    "inputs": [
      {
        "variant": "regression",
        "base_sha": "772ea00fd54bf96499ada83ccbb50d459d5d8a0d",
        "head_sha": "46dd1d0d0ea7ceb0e690203d24fca21c428818ec",
        "patch_sha256": "72b22d36832bb56de0bca98b4a926bafebf6920789a609056d4185a5ae792226",
        "pull_request": "https://github.com/lakeday-org/perch/pull/371",
        "source": "https://github.com/lakeday-org/perch/blob/46dd1d0d0ea7ceb0e690203d24fca21c428818ec/review-evaluation/order_service.py",
        "branch": "eval/order-service-20261014-a"
      },
      {
        "variant": "clean",
        "base_sha": "772ea00fd54bf96499ada83ccbb50d459d5d8a0d",
        "head_sha": "b9ce63ac935739a998ebb40ca6881a261571d208",
        "patch_sha256": "6ffa7e1219e9707ccc592199e8bafde0640b2c7e2eca4129b12be3205f1475a2",
        "pull_request": "https://github.com/lakeday-org/perch/pull/372",
        "source": "https://github.com/lakeday-org/perch/blob/b9ce63ac935739a998ebb40ca6881a261571d208/review-evaluation/order_service.py",
        "branch": "eval/order-service-20261014-b"
      }
    ]
  },
  "cases": [
    {
      "id": "all-stock-lines",
      "method": "can_fulfil",
      "source": "checkout.py",
      "requirement": "Read can_fulfil only. It returns true only when every requested item has sufficient stock; a shortage on any line returns false, including when an earlier line has enough stock.",
      "perch_regression_flagged": true,
      "perch_clean_flagged": false,
      "perch_regression_score": 0.95,
      "perch_floor": 0.5
    },
    {
      "id": "malformed-order-errors",
      "method": "load_order",
      "source": "storage.py",
      "requirement": "Read load_order only. Missing files and malformed JSON propagate an exception to the caller instead of being converted into an empty order.",
      "perch_regression_flagged": true,
      "perch_clean_flagged": false,
      "perch_regression_score": 0.96,
      "perch_floor": 0.5
    },
    {
      "id": "adjacent-expirations",
      "method": "release_expired",
      "source": "inventory.py",
      "requirement": "Read release_expired only. It releases every expired reservation, including adjacent expired entries, removes those entries, and leaves unexpired reservations intact.",
      "perch_regression_flagged": true,
      "perch_clean_flagged": false,
      "perch_regression_score": 0.69,
      "perch_floor": 0.5
    },
    {
      "id": "complete-subtotal",
      "method": "subtotal",
      "source": "cart.py",
      "requirement": "Read subtotal only. It includes price times quantity for every cart line, including the last line, and returns Decimal zero for an empty cart.",
      "perch_regression_flagged": true,
      "perch_clean_flagged": false,
      "perch_regression_score": 0.96,
      "perch_floor": 0.5
    }
  ],
  "protocol": {
    "products": [
      "Perch",
      "CodeRabbit",
      "Greptile"
    ],
    "inputs": "Identical committed base and head code; exact SHA values and patch hashes recorded before reviews.",
    "instructions": "The same four behavior requirements are provided to every reviewer. No clean/broken labels are included in the instruction file or reviewed checkout.",
    "runs": "One regression-suite review and one clean-control review per available service. No repeated runs selected for a better score.",
    "scoring": "For each requirement, report whether the regression was identified and whether the clean control was flagged. Other comments are retained for separate adjudication; unavailable or incomplete reviews are not scored as clean.",
    "cost": "Record observed billed dollars/credits when the service exposes them. Rate-derived estimates and trial charges remain separate; unavailable bill data remains null.",
    "time": "Wall time from review command start to final completion, excluding install, authentication and repository indexing.",
    "settings": "Default effort; Perch jev-latest with the four scoped file rules; CodeRabbit committed agent review with shared instructions; Greptile base effort with identical shared instructions. No paid overage flags.",
    "limits": "This is a focused disclosed-requirement suite, not a representative all-bug accuracy benchmark or a complete security evaluation. Fresh judgments can vary. Different review surfaces and model choices are reported."
  },
  "products": [
    {
      "name": "Perch",
      "status": "completed",
      "version": "DEVELOPMENT (4af67af)",
      "model": "jev-latest",
      "endpoint": "https://api.typesafe.ai/v1/systemone",
      "regressions_flagged": 4,
      "regressions_total": 4,
      "clean_requirements_flagged": 0,
      "clean_requirements_total": 4,
      "runtime_seconds": {
        "regression": 0.436,
        "clean": 0.389
      },
      "actual_billed_usd": null,
      "rate_derived_customer_estimate_usd": 0.0008,
      "cost_note": "Published customer-rate estimate for eight fresh rule checks, not a billing receipt. The CLI used its configured direct SystemOne endpoint; timing describes that path and actual billed cost is unavailable."
    },
    {
      "name": "CodeRabbit",
      "status": "not_run",
      "version": "0.9.0",
      "regressions_flagged": null,
      "clean_requirements_flagged": null,
      "runtime_seconds": null,
      "actual_billed_usd": null,
      "reason": "The CLI is installed, but an authenticated review session is not available. No review result or comparative score is claimed."
    },
    {
      "name": "Greptile",
      "status": "not_run",
      "version": "3.6.2",
      "regressions_flagged": null,
      "clean_requirements_flagged": null,
      "runtime_seconds": null,
      "actual_billed_usd": null,
      "reason": "The CLI is installed, but an authenticated review session is not available. No review result or comparative score is claimed."
    }
  ],
  "interpretation": "Four disclosed behavior requirements on one Python helper file. Perch results are complete; unavailable competitors are not counted as misses. This suite is not representative all-bug accuracy or proof of comparative superiority."
}