{
  "slug": "research",
  "role": "ML research",
  "title": "Observed outcome contrast",
  "outcome": "Counted missing outcomes in the denominator; both checks failed.",
  "caseId": "holdout-research-faulty",
  "recordedAt": "18 October 2026",
  "task": "Input has device and sham lists of binary outcomes (0 or 1) or null for missing. Return device-minus-sham risk difference using only observed outcomes in each arm. Missing values must not count as failures. If either arm has no observed outcomes return null. This is an observed-outcome contrast, not a causal claim.",
  "filename": "solution.py",
  "code": "def solve(payload):\n    def rate(values):\n        observed = [value for value in values if value is not None]\n        if not observed:\n            return None\n        return sum(observed) / len(values)\n    device, sham = rate(payload[\"device\"]), rate(payload[\"sham\"])\n    if device is None or sham is None:\n        return None\n    return device - sham",
  "checks": [
    {
      "id": "probe-0",
      "input": {
        "device": [
          1,
          1,
          null
        ],
        "sham": [
          1,
          0,
          0,
          null
        ]
      },
      "expected": 0.6666666666666666,
      "passed": false
    },
    {
      "id": "probe-1",
      "input": {
        "device": [
          1,
          0,
          null,
          null
        ],
        "sham": [
          1,
          0
        ]
      },
      "expected": 0.0,
      "passed": false
    }
  ],
  "reviewSummary": "The implementation excludes missing values from the numerator but not the denominator, so mixed observed/missing inputs can yield incorrect contrasts. Empty-observation handling and input immutability are supported by source inspection. No execution evidence was supplied.",
  "findings": [
    {
      "criterion": "The implementation computes the required result for all stated inputs.",
      "assessment": "Contradicted",
      "evidence": [
        {
          "claim": "The numerator excludes nulls, but the denominator includes them. For device=[1,null], sham=[0], the code returns 0.5 instead of the required 1.",
          "line": 3,
          "path": "solution.py"
        },
        {
          "claim": "The subtraction order is correct, but its arm rates use total list lengths. Thus missing outcomes can change the required contrast.",
          "line": 6,
          "path": "solution.py"
        }
      ]
    },
    {
      "criterion": "The implementation leaves caller-owned input lists, records and vectors unchanged.",
      "assessment": "Supported",
      "evidence": [
        {
          "claim": "The function only reads payload entries and iterates their lists. Filtering creates a new list; no assignment or mutating operation targets caller-owned objects.",
          "line": 1,
          "path": "solution.py"
        }
      ]
    }
  ],
  "provenance": {
    "reportSha256": "840b0ce50586d117327697dad955f34282ecc9a68cbb66f106e55cb85a72e0e6",
    "requestSha256": "f88250fddf3a2d4049abec30f670b6c2df5b00511ff693e312b3f85320c5771e",
    "sourceSha256": "ea372e54dd7698d46082d15433509ef902fc1b8682ae250a62c25728b28e9c58"
  }
}