{
  "slug": "ml",
  "role": "ML engineering",
  "title": "Depth-scaled attention weights",
  "outcome": "Omitted attention scaling; both checks failed.",
  "caseId": "holdout-ml-faulty",
  "recordedAt": "18 October 2026",
  "task": "Input has a nonempty query vector q of depth D and a nonempty list k of key vectors of the same depth. Return key probabilities softmax(dot(q,k)/sqrt(D)). Use a maximum shift before exponentiation. All dot products are finite and within [-100,100]. Do not mutate input vectors.",
  "filename": "solution.py",
  "code": "import math\ndef solve(payload):\n    q, keys = payload[\"q\"], payload[\"k\"]\n    scale = 1.0\n    logits = [sum(a*b for a, b in zip(q, key)) / scale for key in keys]\n    maximum = max(logits)\n    exps = [math.exp(value - maximum) for value in logits]\n    total = sum(exps)\n    return [value / total for value in exps]",
  "checks": [
    {
      "id": "probe-0",
      "input": {
        "k": [
          [
            2,
            0,
            0,
            0
          ],
          [
            0,
            0,
            0,
            0
          ]
        ],
        "q": [
          2,
          0,
          0,
          0
        ]
      },
      "expected": [
        0.8807970779778823,
        0.11920292202211755
      ],
      "passed": false
    },
    {
      "id": "probe-1",
      "input": {
        "k": [
          [
            0,
            0,
            0,
            -2
          ],
          [
            0,
            0,
            0,
            0
          ]
        ],
        "q": [
          0,
          0,
          0,
          3
        ]
      },
      "expected": [
        0.04742587317756678,
        0.9525741268224334
      ],
      "passed": false
    }
  ],
  "reviewSummary": "The implementation uses maximum-shifted softmax and leaves inputs untouched, but omits depth scaling, producing incorrect probabilities for general depths greater than one. No execution records were supplied.",
  "findings": [
    {
      "criterion": "The implementation computes the required result for all stated inputs.",
      "assessment": "Contradicted",
      "evidence": [
        {
          "claim": "Scale is fixed at 1. For q=[1,0] and keys=[[1,0],[0,0]], logits are [1,0] instead of [1/sqrt(2),0], yielding different probabilities despite valid finite dot products.",
          "line": 3,
          "path": "solution.py"
        },
        {
          "claim": "Exponentials are normalized in key order, but normalization uses unscaled logits. The valid depth-two example therefore returns approximately [0.7311,0.2689], not [0.6698,0.3302].",
          "line": 4,
          "path": "solution.py"
        }
      ]
    },
    {
      "criterion": "The implementation leaves caller-owned input lists, records and vectors unchanged.",
      "assessment": "Supported",
      "evidence": [
        {
          "claim": "The function only reads payload entries and iterates over vectors. It constructs new logits, exponentials, and result lists without assigning into caller-owned objects.",
          "line": 2,
          "path": "solution.py"
        }
      ]
    }
  ],
  "provenance": {
    "reportSha256": "f76549e726a40e5b7425093a5135597cfc3cddd3846e094617a77b2097945478",
    "requestSha256": "0ebe4da0ba536d22c9942dd9d3c1fdb7f82dca01a504123f90894b4239d04f74",
    "sourceSha256": "480f0b550fb34afa5ff143631841f0d7a4c0c498abaab709b62f8f74f8cc0c13"
  }
}