{
  "schemaVersion": 1,
  "suiteHash": "608d81e3473fa321783e5d77a56b8cb0210a6a2493cbb3f85a68eb319b4789df",
  "profiles": [
    {
      "id": "transform-fidelity",
      "name": "Transform fidelity",
      "description": "Follow exact text and symbol transformation contracts.",
      "cells": [
        "echo:t0",
        "strings:t1",
        "strings:t2",
        "cipher:t2"
      ]
    },
    {
      "id": "symbolic-execution",
      "name": "Symbolic execution",
      "description": "Carry deterministic arithmetic, machine-state, and automaton steps.",
      "cells": [
        "arithmetic:t0",
        "arithmetic:t1",
        "arithmetic:t2",
        "registervm:t3",
        "registervm:t4",
        "automata:t4",
        "automata:t5"
      ]
    },
    {
      "id": "constraint-solving",
      "name": "Constraint solving",
      "description": "Satisfy ordering, path, logic, Sudoku, and alphametic constraints.",
      "cells": [
        "ordering:t3",
        "gridpath:t3",
        "knights:t3",
        "knights:t4",
        "sudoku:t3",
        "sudoku:t4",
        "sudoku:t5",
        "cryptarithm:t4",
        "cryptarithm:t5"
      ]
    },
    {
      "id": "rule-induction",
      "name": "Rule induction",
      "description": "Infer a declared finite rule from examples and apply it to a new input.",
      "cells": [
        "sequence:t2",
        "hiddenfn:t5",
        "hiddenfn:t6",
        "gridxf:t5",
        "gridxf:t6"
      ]
    }
  ],
  "models": [
    {
      "model": "openai/gpt-5-mini",
      "profiles": {
        "transform-fidelity": {
          "total": 80,
          "n": 80,
          "errors": 0,
          "passed": 79,
          "strict": 0.9875,
          "interval95": [
            0.9325372992233133,
            0.9977900244795476
          ],
          "finalAnswerMatches": 79,
          "finalAnswer": 0.9875
        },
        "symbolic-execution": {
          "total": 140,
          "n": 139,
          "errors": 1,
          "passed": 125,
          "strict": 0.8992805755395683,
          "interval95": [
            0.8380337291658037,
            0.9390515856750085
          ],
          "finalAnswerMatches": 125,
          "finalAnswer": 0.8992805755395683
        },
        "constraint-solving": {
          "total": 180,
          "n": 180,
          "errors": 0,
          "passed": 142,
          "strict": 0.7888888888888889,
          "interval95": [
            0.7235528134620787,
            0.8421520099609893
          ],
          "finalAnswerMatches": 142,
          "finalAnswer": 0.7888888888888889
        },
        "rule-induction": {
          "total": 100,
          "n": 100,
          "errors": 0,
          "passed": 91,
          "strict": 0.91,
          "interval95": [
            0.837737871472837,
            0.9519274599974348
          ],
          "finalAnswerMatches": 91,
          "finalAnswer": 0.91
        }
      }
    },
    {
      "model": "google/gemini-2.5-pro",
      "profiles": {
        "transform-fidelity": {
          "total": 80,
          "n": 80,
          "errors": 0,
          "passed": 71,
          "strict": 0.8875,
          "interval95": [
            0.7998178190762713,
            0.9396731305336952
          ],
          "finalAnswerMatches": 71,
          "finalAnswer": 0.8875
        },
        "symbolic-execution": {
          "total": 140,
          "n": 140,
          "errors": 0,
          "passed": 129,
          "strict": 0.9214285714285714,
          "interval95": [
            0.8647850070923281,
            0.9555626240248808
          ],
          "finalAnswerMatches": 129,
          "finalAnswer": 0.9214285714285714
        },
        "constraint-solving": {
          "total": 180,
          "n": 180,
          "errors": 0,
          "passed": 121,
          "strict": 0.6722222222222223,
          "interval95": [
            0.6006746450320746,
            0.7365724612393698
          ],
          "finalAnswerMatches": 132,
          "finalAnswer": 0.7333333333333333
        },
        "rule-induction": {
          "total": 100,
          "n": 100,
          "errors": 0,
          "passed": 82,
          "strict": 0.82,
          "interval95": [
            0.7333264161369926,
            0.8829977450105364
          ],
          "finalAnswerMatches": 83,
          "finalAnswer": 0.83
        }
      }
    },
    {
      "model": "openai/gpt-4.1",
      "profiles": {
        "transform-fidelity": {
          "total": 80,
          "n": 80,
          "errors": 0,
          "passed": 45,
          "strict": 0.5625,
          "interval95": [
            0.4534113429694972,
            0.6658613908385618
          ],
          "finalAnswerMatches": 45,
          "finalAnswer": 0.5625
        },
        "symbolic-execution": {
          "total": 140,
          "n": 140,
          "errors": 0,
          "passed": 77,
          "strict": 0.55,
          "interval95": [
            0.4673531431737411,
            0.6299762367893177
          ],
          "finalAnswerMatches": 78,
          "finalAnswer": 0.5571428571428572
        },
        "constraint-solving": {
          "total": 180,
          "n": 180,
          "errors": 0,
          "passed": 42,
          "strict": 0.23333333333333334,
          "interval95": [
            0.17751311260089475,
            0.3002978196239655
          ],
          "finalAnswerMatches": 82,
          "finalAnswer": 0.45555555555555555
        },
        "rule-induction": {
          "total": 100,
          "n": 100,
          "errors": 0,
          "passed": 41,
          "strict": 0.41,
          "interval95": [
            0.3186731302113651,
            0.5079856994658921
          ],
          "finalAnswerMatches": 41,
          "finalAnswer": 0.41
        }
      }
    },
    {
      "model": "openai/gpt-4o-mini",
      "profiles": {
        "transform-fidelity": {
          "total": 80,
          "n": 80,
          "errors": 0,
          "passed": 40,
          "strict": 0.5,
          "interval95": [
            0.3929741508611972,
            0.6070258491388028
          ],
          "finalAnswerMatches": 40,
          "finalAnswer": 0.5
        },
        "symbolic-execution": {
          "total": 140,
          "n": 140,
          "errors": 0,
          "passed": 50,
          "strict": 0.35714285714285715,
          "interval95": [
            0.2825610827114282,
            0.4393549745369755
          ],
          "finalAnswerMatches": 52,
          "finalAnswer": 0.37142857142857144
        },
        "constraint-solving": {
          "total": 180,
          "n": 180,
          "errors": 0,
          "passed": 28,
          "strict": 0.15555555555555556,
          "interval95": [
            0.10987007034797208,
            0.2156357171091391
          ],
          "finalAnswerMatches": 49,
          "finalAnswer": 0.2722222222222222
        },
        "rule-induction": {
          "total": 100,
          "n": 100,
          "errors": 0,
          "passed": 22,
          "strict": 0.22,
          "interval95": [
            0.150012823846054,
            0.31070353514985793
          ],
          "finalAnswerMatches": 26,
          "finalAnswer": 0.26
        }
      }
    },
    {
      "model": "meta/llama-3.1-8b",
      "profiles": {
        "transform-fidelity": {
          "total": 80,
          "n": 80,
          "errors": 0,
          "passed": 20,
          "strict": 0.25,
          "interval95": [
            0.16806231857542375,
            0.3548467461923397
          ],
          "finalAnswerMatches": 20,
          "finalAnswer": 0.25
        },
        "symbolic-execution": {
          "total": 140,
          "n": 140,
          "errors": 0,
          "passed": 20,
          "strict": 0.14285714285714285,
          "interval95": [
            0.09441991250112675,
            0.21037023061988253
          ],
          "finalAnswerMatches": 20,
          "finalAnswer": 0.14285714285714285
        },
        "constraint-solving": {
          "total": 180,
          "n": 180,
          "errors": 0,
          "passed": 2,
          "strict": 0.011111111111111112,
          "interval95": [
            0.0030523748459926198,
            0.03960100089958458
          ],
          "finalAnswerMatches": 3,
          "finalAnswer": 0.016666666666666666
        },
        "rule-induction": {
          "total": 100,
          "n": 100,
          "errors": 0,
          "passed": 5,
          "strict": 0.05,
          "interval95": [
            0.02154367915436796,
            0.11175046923191913
          ],
          "finalAnswerMatches": 6,
          "finalAnswer": 0.06
        }
      }
    }
  ]
}