{ "meta": { "step": 1637, "version": "invariance-empirical-v0.1", "startedAt": "2026-08-31T15:20:11.451Z", "finishedAt": "2026-08-31T15:30:10.068Z", "model": "qwen2.5:3b", "samplesPerCondition": 2, "alpha": 0.05, "fdrQ": 0.05, "corpus": { "version": "arithmetic-v1", "count": 12, "difficulties": { "easy": 6, "medium": 6 }, "answerRange": [ 5, 200 ] }, "transforms": [ { "id": "variable-rename", "description": "Deterministically substitute personal names (Alice→Zara etc.) without changing quantities" }, { "id": "whitespace-noise", "description": "Add extra whitespace between sentences and after commas; meaning unchanged" }, { "id": "synonym-swap", "description": "Replace verbs and common nouns with synonyms; quantities and numeric answer preserved" } ], "comparator": "numericMatch(absTolerance=1e-6, extractLastNumber)", "honestScope": [ "Single local model (Ollama), single run, deterministic mechanical transforms.", "Comparator only extracts the last integer; models that write the answer earlier are penalized (comparator brittleness = artifact, not model failure).", "Statistical layer uses one-tailed exact binomial against H0: equivalence rate >= 0.5.", "Cross-problem BH-FDR aggregate uses each (problem × transform) as an independent trial (independence assumption).", "Transformation \"equivalence\" is a caller assertion (asserter=self); not formally verified." ] }, "perProblem": [ { "problemId": "a01", "expected": 15, "difficulty": "easy", "originalText": "Alice has 12 apples. She gives 5 to Bob and buys 8 more. How many apples does Alice have now?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\).\nThen she buys 8 more: \\(7 + 8 = 15\\).\n(final line)\n15", "latencyMs": 15577 }, "after": { "text": "(brief work)\nZara starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\).\nThen she buys 8 more apples: \\(7 + 8 = 15\\).\n(final answer) <15>", "latencyMs": 8241 } }, { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\) apples.\nThen she buys 8 more: \\(7 + 8 = 15\\) apples.\n(final line)\n15", "latencyMs": 6534 }, "after": { "text": "(brief work)\nZara starts with 12 apples.\nShe gives away 5 apples, so she has \\(12 - 5 = 7\\) apples left.\nThen she buys 8 more apples, so she now has \\(7 + 8 = 15\\) apples.\n\n<15>", "latencyMs": 7613 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\).\nThen she buys 8 more: \\(7 + 8 = 15\\).\n(final line)\n15", "latencyMs": 15577 }, "after": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\) apples.\nThen she buys 8 more: \\(7 + 8 = 15\\) apples.\n(final line)\n15", "latencyMs": 6935 } }, { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\) apples.\nThen she buys 8 more: \\(7 + 8 = 15\\) apples.\n(final line)\n15", "latencyMs": 6534 }, "after": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\)\nThen she buys 8 more: \\(7 + 8 = 15\\)\n(final line)\n15", "latencyMs": 5972 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\).\nThen she buys 8 more: \\(7 + 8 = 15\\).\n(final line)\n15", "latencyMs": 15577 }, "after": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives 5 to Bob: \\(12 - 5 = 7\\).\nThen she buys 8 more: \\(7 + 8 = 15\\).\nThe final number of apples Alice has is 15.\n\n<15>", "latencyMs": 8988 } }, { "before": { "text": "(brief work)\nAlice starts with 12 apples.\nShe gives away 5 apples: \\(12 - 5 = 7\\) apples.\nThen she buys 8 more: \\(7 + 8 = 15\\) apples.\n(final line)\n15", "latencyMs": 6534 }, "after": { "text": "(brief work)\nAlice initially has 12 apples.\nShe gives 5 to Bob: \\(12 - 5 = 7\\) apples.\nThen she buys 8 more: \\(7 + 8 = 15\\) apples.\n(final line)\n15", "latencyMs": 7162 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a02", "expected": 28, "difficulty": "easy", "originalText": "A book costs 7 dollars. Charlie buys 4 books. How much does Charlie pay in total?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "Brief work:\nEach book costs $7, and Charlie buys 4 books.\nTotal cost = 7 * 4 = 28 dollars.\n\nAnswer: <28>", "latencyMs": 5228 }, "after": { "text": "Brief work:\nThe cost of one book is $7.\nXander buys 4 books.\nTotal cost = 7 dollars/book * 4 books = 28 dollars.\n\nAnswer: 28", "latencyMs": 6185 } }, { "before": { "text": "Brief work:\nEach book costs $7 and Charlie buys 4 books. Therefore, the total cost is calculated as follows:\n\\[ 7 \\text{ dollars/book} \\times 4 \\text{ books} = 28 \\text{ dollars} \\]\n\nAnswer:\n\\[ 28 \\]", "latencyMs": 7340 }, "after": { "text": "Brief work:\nThe cost of one book is $7. Xander buys 4 books.\nTotal cost = 7 * 4 = 28 dollars.\n\nAnswer:\n28", "latencyMs": 4270 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "Brief work:\nEach book costs $7, and Charlie buys 4 books.\nTotal cost = 7 * 4 = 28 dollars.\n\nAnswer: <28>", "latencyMs": 5228 }, "after": { "text": "(7 dollars/book * 4 books) = 28 dollars \n28", "latencyMs": 3732 } }, { "before": { "text": "Brief work:\nEach book costs $7 and Charlie buys 4 books. Therefore, the total cost is calculated as follows:\n\\[ 7 \\text{ dollars/book} \\times 4 \\text{ books} = 28 \\text{ dollars} \\]\n\nAnswer:\n\\[ 28 \\]", "latencyMs": 7340 }, "after": { "text": "(7 dollars/book * 4 books) = 28 dollars \n<28>", "latencyMs": 2355 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "Brief work:\nEach book costs $7, and Charlie buys 4 books.\nTotal cost = 7 * 4 = 28 dollars.\n\nAnswer: <28>", "latencyMs": 5228 }, "after": { "text": "Brief work:\nEach book costs $7, and Charlie purchases 4 books. The total cost is calculated as:\n\\[ 7 \\times 4 = 28 \\]\n\nAnswer:\n<28>", "latencyMs": 6601 } }, { "before": { "text": "Brief work:\nEach book costs $7 and Charlie buys 4 books. Therefore, the total cost is calculated as follows:\n\\[ 7 \\text{ dollars/book} \\times 4 \\text{ books} = 28 \\text{ dollars} \\]\n\nAnswer:\n\\[ 28 \\]", "latencyMs": 7340 }, "after": { "text": "Brief work:\nCost of one book = $7\nNumber of books = 4\nTotal cost = Cost of one book × Number of books = $7 × 4 = $28\n\nAnswer: <28>", "latencyMs": 5673 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a03", "expected": 18, "difficulty": "easy", "originalText": "Emma runs 3 miles every day for 6 days. How many miles does Emma run in total?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "Brief work:\nEmma runs 3 miles per day for 6 days. Therefore, the total miles run is \\(3 \\times 6 = 18\\) miles.\n\nAnswer:\n18", "latencyMs": 6309 }, "after": { "text": "Brief work:\nVera runs 3 miles each day for 6 days, so the total miles run can be calculated as:\n\\[ 3 \\, \\text{miles/day} \\times 6 \\, \\text{days} = 18 \\, \\text{miles} \\]\n\nAnswer:\n\\[ 18 \\]", "latencyMs": 9507 } }, { "before": { "text": "Brief work:\nEmma runs 3 miles each day for 6 days. So the total miles can be calculated as:\n\\[ 3 \\, \\text{miles/day} \\times 6 \\, \\text{days} = 18 \\, \\text{miles} \\]\n\nAnswer:\n\\[ 18 \\]", "latencyMs": 8069 }, "after": { "text": "Brief work:\nVera runs 3 miles every day for 6 days. \nTotal miles = 3 miles/day * 6 days = 18 miles\n\nAnswer:\n18", "latencyMs": 4804 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "Brief work:\nEmma runs 3 miles per day for 6 days. Therefore, the total miles run is \\(3 \\times 6 = 18\\) miles.\n\nAnswer:\n18", "latencyMs": 6309 }, "after": { "text": "Brief Work:\nEmma runs 3 miles every day for 6 days, so the total miles she runs is \\(3 \\times 6 = 18\\).\n\nAnswer:\n18", "latencyMs": 6428 } }, { "before": { "text": "Brief work:\nEmma runs 3 miles each day for 6 days. So the total miles can be calculated as:\n\\[ 3 \\, \\text{miles/day} \\times 6 \\, \\text{days} = 18 \\, \\text{miles} \\]\n\nAnswer:\n\\[ 18 \\]", "latencyMs": 8069 }, "after": { "text": "Brief work:\nEmma runs 3 miles each day for 6 days. \nTotal miles = 3 miles/day * 6 days = 18 miles\n\nAnswer:\n18", "latencyMs": 4761 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "Brief work:\nEmma runs 3 miles per day for 6 days. Therefore, the total miles run is \\(3 \\times 6 = 18\\) miles.\n\nAnswer:\n18", "latencyMs": 6309 }, "after": { "text": "Brief work:\nEmma jogs 3 miles each day for 6 days.\nTotal miles = 3 miles/day * 6 days = 18 miles.\n\nAnswer:\n18", "latencyMs": 7052 } }, { "before": { "text": "Brief work:\nEmma runs 3 miles each day for 6 days. So the total miles can be calculated as:\n\\[ 3 \\, \\text{miles/day} \\times 6 \\, \\text{days} = 18 \\, \\text{miles} \\]\n\nAnswer:\n\\[ 18 \\]", "latencyMs": 8069 }, "after": { "text": "Brief work:\nEmma jogs 3 miles each day for 6 days. So, the total miles she runs is 3 * 6 = 18 miles.\n\nAnswer:\n18", "latencyMs": 5583 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a04", "expected": 120, "difficulty": "easy", "originalText": "A school has 24 students in each classroom and 5 classrooms. How many students are in the school?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n\n<120>", "latencyMs": 4325 }, "after": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n\n<120>", "latencyMs": 3333 } }, { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n120", "latencyMs": 3457 }, "after": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n120", "latencyMs": 3348 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n\n<120>", "latencyMs": 4325 }, "after": { "text": "(24 students/classroom * 5 classrooms) = 120 students \n<120>", "latencyMs": 3715 } }, { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n120", "latencyMs": 3457 }, "after": { "text": "(Work): Each classroom has 24 students, and there are 5 classrooms. Therefore, the total number of students is \\( 24 \\times 5 = 120 \\).\n\n(integer) 120", "latencyMs": 5012 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n\n<120>", "latencyMs": 4325 }, "after": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n\n<120>", "latencyMs": 3607 } }, { "before": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n120", "latencyMs": 3457 }, "after": { "text": "(24 students/classroom * 5 classrooms) = 120 students\n<120>", "latencyMs": 2693 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a05", "expected": 30, "difficulty": "medium", "originalText": "David had 50 marbles. He lost 12 in the park and gave 8 to his sister. How many marbles does David have left?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(brief work)\n50 - 12 = 38\n38 - 8 = 30\n\n\n30", "latencyMs": 4757 }, "after": { "text": "(brief work)\n50 - 12 = 38\n38 - 8 = 30\n\n30", "latencyMs": 4880 } }, { "before": { "text": "(brief work)\nDavid initially had 50 marbles.\nHe lost 12 marbles, so he had \\(50 - 12 = 38\\) marbles left.\nThen he gave 8 marbles to his sister, so he had \\(38 - 8 = 30\\) marbles left.\n\n(integer) 30", "latencyMs": 7702 }, "after": { "text": "(brief work)\nWren started with 50 marbles.\nLost 12 marbles: \\(50 - 12 = 38\\)\nGave 8 marbles to his sister: \\(38 - 8 = 30\\)\n\nAnswer: <30>", "latencyMs": 6680 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(brief work)\n50 - 12 = 38\n38 - 8 = 30\n\n\n30", "latencyMs": 4757 }, "after": { "text": "(brief work)\n50 - 12 = 38\n38 - 8 = 30\n\n<30>", "latencyMs": 4717 } }, { "before": { "text": "(brief work)\nDavid initially had 50 marbles.\nHe lost 12 marbles, so he had \\(50 - 12 = 38\\) marbles left.\nThen he gave 8 marbles to his sister, so he had \\(38 - 8 = 30\\) marbles left.\n\n(integer) 30", "latencyMs": 7702 }, "after": { "text": "(brief work)\nDavid initially had 50 marbles.\nHe lost 12 marbles, so he had \\(50 - 12 = 38\\) marbles left.\nHe then gave 8 marbles to his sister, so he had \\(38 - 8 = 30\\) marbles left.\n\n(integer) 30", "latencyMs": 8184 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(brief work)\n50 - 12 = 38\n38 - 8 = 30\n\n\n30", "latencyMs": 4757 }, "after": { "text": "(brief work)\nDavid initially had 50 marbles.\nHe lost 12, so he had 50 - 12 = 38 marbles left.\nHe then handed over 8 to his sister, so he had 38 - 8 = 30 marbles left.\n(final line)\n<30>", "latencyMs": 10421 } }, { "before": { "text": "(brief work)\nDavid initially had 50 marbles.\nHe lost 12 marbles, so he had \\(50 - 12 = 38\\) marbles left.\nThen he gave 8 marbles to his sister, so he had \\(38 - 8 = 30\\) marbles left.\n\n(integer) 30", "latencyMs": 7702 }, "after": { "text": "(brief work)\nDavid started with 50 marbles.\nHe lost 12, so he had 50 - 12 = 38 marbles left.\nHe then handed over 8 to his sister, so he had 38 - 8 = 30 marbles left.\n(final line)\n30", "latencyMs": 7281 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a06", "expected": 5, "difficulty": "medium", "originalText": "A store sells shirts for 15 dollars each. Fiona buys 3 shirts and pays with a 50-dollar bill. How much change does Fiona receive?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "Brief work:\nCost of 3 shirts = 15 * 3 = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 - 45 = 5 dollars\n\nAnswer:\n5", "latencyMs": 6941 }, "after": { "text": "Brief work:\n1. Cost of 3 shirts: 15 * 3 = 45 dollars\n2. Change received: 50 - 45 = 5 dollars\n\nAnswer: 5", "latencyMs": 6187 } }, { "before": { "text": "Brief work:\nCost of 3 shirts = 15 dollars/shirt × 3 shirts = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 dollars - 45 dollars = 5 dollars\n\nAnswer: 5", "latencyMs": 6321 }, "after": { "text": "Brief work:\n1. Cost of 3 shirts = 3 * 15 = 45 dollars\n2. Change received = 50 - 45 = 5 dollars\n\nAnswer:\n5", "latencyMs": 4535 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "Brief work:\nCost of 3 shirts = 15 * 3 = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 - 45 = 5 dollars\n\nAnswer:\n5", "latencyMs": 6941 }, "after": { "text": "Brief work:\n1. Cost of 3 shirts = 15 * 3 = 45 dollars.\n2. Change received = 50 - 45 = 5 dollars.\n\nAnswer: 5", "latencyMs": 6132 } }, { "before": { "text": "Brief work:\nCost of 3 shirts = 15 dollars/shirt × 3 shirts = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 dollars - 45 dollars = 5 dollars\n\nAnswer: 5", "latencyMs": 6321 }, "after": { "text": "Brief work:\n1. Cost of 3 shirts = 3 * $15 = $45\n2. Change received = $50 - $45 = $5\n\nAnswer: 5", "latencyMs": 4535 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "Brief work:\nCost of 3 shirts = 15 * 3 = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 - 45 = 5 dollars\n\nAnswer:\n5", "latencyMs": 6941 }, "after": { "text": "Brief work:\n1 T-shirt cost: $15\nFiona bought: 3 T-shirts\nTotal cost: 15 * 3 = $45\nFiona gave: $50\nChange received: 50 - 45 = $5\n\n\n5", "latencyMs": 8975 } }, { "before": { "text": "Brief work:\nCost of 3 shirts = 15 dollars/shirt × 3 shirts = 45 dollars\nAmount paid = 50 dollars\nChange received = Amount paid - Cost of shirts = 50 dollars - 45 dollars = 5 dollars\n\nAnswer: 5", "latencyMs": 6321 }, "after": { "text": "Brief work:\n1. The cost of one T-shirt is 15 dollars.\n2. Fiona buys 3 T-shirts, so the total cost is \\(3 \\times 15 = 45\\) dollars.\n3. Fiona spends with a 50-dollar bill, so the change she receives is \\(50 - 45 = 5\\) dollars.\n\nAnswer: <5>", "latencyMs": 8615 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a07", "expected": 168, "difficulty": "medium", "originalText": "A garden has 4 rows of tomato plants, with 7 plants in each row. If each plant produces 6 tomatoes, how many tomatoes are there in the garden?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = (28 * 6) = 168\n<168>", "latencyMs": 5115 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = (28 * 6) = 168\n<168>", "latencyMs": 3909 } }, { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant \n<672>", "latencyMs": 2500 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant \n<672>", "latencyMs": 2272 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = (28 * 6) = 168\n<168>", "latencyMs": 5115 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = (28 * 6) = 168 \n<168>", "latencyMs": 5033 } }, { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant \n<672>", "latencyMs": 2500 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant\n 168", "latencyMs": 2772 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 0.75, "equivalentCount": 1 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = (28 * 6) = 168\n<168>", "latencyMs": 5115 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant = <168>", "latencyMs": 4135 } }, { "before": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant \n<672>", "latencyMs": 2500 }, "after": { "text": "(4 rows * 7 plants/row) * 6 tomatoes/plant =\n<168>", "latencyMs": 2579 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 0.75, "equivalentCount": 1 } ] }, { "problemId": "a08", "expected": 200, "difficulty": "easy", "originalText": "George reads 25 pages of a book each night. How many pages will George have read after 8 nights?", "aggregate": { "verdict": { "name": "BOTH", "symbol": "B", "numeric": 2 }, "reason": "partial_invariance", "trueCount": 2, "falseCount": 1, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "Brief work:\nEach night, George reads 25 pages.\nOver 8 nights, the total number of pages read is 25 * 8.\n\nAnswer:\n200", "latencyMs": 5076 }, "after": { "text": "Brief work:\nToma reads 25 pages each night. Over 8 nights, the total number of pages read is calculated as:\n25 pages/night * 8 nights = 200 pages\n\nThe answer is: <200>", "latencyMs": 7361 } }, { "before": { "text": "Brief work:\nEach night George reads 25 pages. Over 8 nights, the total number of pages read can be calculated as:\n\\[ 25 \\text{ pages/night} \\times 8 \\text{ nights} = 200 \\text{ pages} \\]\n\nAnswer:\n\\[ 200 \\]", "latencyMs": 7828 }, "after": { "text": "Brief work:\nToma reads 25 pages each night, and there are 8 nights.\nTotal pages read = 25 pages/night * 8 nights\n\nAnswer:\n100", "latencyMs": 5414 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 0.75, "equivalentCount": 1 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "Brief work:\nEach night, George reads 25 pages.\nOver 8 nights, the total number of pages read is 25 * 8.\n\nAnswer:\n200", "latencyMs": 5076 }, "after": { "text": "Brief work:\nEach night, George reads 25 pages.\nOver 8 nights, the total number of pages read is 25 * 8.\n\nAnswer: 200", "latencyMs": 5877 } }, { "before": { "text": "Brief work:\nEach night George reads 25 pages. Over 8 nights, the total number of pages read can be calculated as:\n\\[ 25 \\text{ pages/night} \\times 8 \\text{ nights} = 200 \\text{ pages} \\]\n\nAnswer:\n\\[ 200 \\]", "latencyMs": 7828 }, "after": { "text": "Brief work:\nEach night George reads 25 pages. Over 8 nights, the total number of pages read can be calculated as:\n\\[ 25 \\text{ pages/night} \\times 8 \\text{ nights} = 200 \\text{ pages} \\]\n\nAnswer: <200>", "latencyMs": 8411 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "Brief work:\nEach night, George reads 25 pages.\nOver 8 nights, the total number of pages read is 25 * 8.\n\nAnswer:\n200", "latencyMs": 5076 }, "after": { "text": "Brief work:\nEach night, George reads 25 pages.\nOver 8 nights, the total number of pages read is 25 * 8.\n\nAnswer:\n100", "latencyMs": 5661 } }, { "before": { "text": "Brief work:\nEach night George reads 25 pages. Over 8 nights, the total number of pages read can be calculated as:\n\\[ 25 \\text{ pages/night} \\times 8 \\text{ nights} = 200 \\text{ pages} \\]\n\nAnswer:\n\\[ 200 \\]", "latencyMs": 7828 }, "after": { "text": "Brief work:\nGeorge reads 25 pages each night. Over 8 nights, the total number of pages read can be calculated as:\n\\[ 25 \\text{ pages/night} \\times 8 \\text{ nights} = 200 \\text{ pages} \\]\n\nAnswer:\n\\[ 200 \\]", "latencyMs": 8074 } } ], "singleVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "singleReason": "invariance_violated:numeric_mismatch(diff=1.000e+2)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 0.75, "equivalentCount": 1 } ] }, { "problemId": "a09", "expected": 12, "difficulty": "easy", "originalText": "A bakery baked 96 cookies and packed them equally into 8 boxes. How many cookies are in each box?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 3823 }, "after": { "text": "(96 ÷ 8 = 12)\n<12>", "latencyMs": 2050 } }, { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 2646 }, "after": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 2582 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 3823 }, "after": { "text": "Brief work:\nTo find out how many cookies are in each box, divide the total number of cookies by the number of boxes:\n\\[ \\frac{96}{8} = 12 \\]\n\nAnswer: <12>", "latencyMs": 7277 } }, { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 2646 }, "after": { "text": "Brief work:\nTo find out how many cookies are in each box, divide the total number of cookies by the number of boxes.\n\\[ \\text{Cookies per box} = \\frac{96 \\text{ cookies}}{8 \\text{ boxes}} \\]\n\nAnswer:\n<12>", "latencyMs": 7233 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 3823 }, "after": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 3653 } }, { "before": { "text": "(96 ÷ 8 = 12) then on the final line: 12", "latencyMs": 2646 }, "after": { "text": "(96 ÷ 8 = 12) \n<12>", "latencyMs": 2088 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a10", "expected": 108, "difficulty": "medium", "originalText": "Hannah earns 9 dollars per hour babysitting. If she babysits for 7 hours on Saturday and 5 hours on Sunday, how much does she earn in total for the weekend?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(9 dollars/hour * 7 hours for Saturday) + (9 dollars/hour * 5 hours for Sunday) = 63 + 45 = 108 dollars\n\n<108>", "latencyMs": 7121 }, "after": { "text": "(Work)\nSena earns $9 per hour.\nOn Saturday, she earns $9 * 7 = $63.\nOn Sunday, she earns $9 * 5 = $45.\nTotal earnings for the weekend = $63 + $45 = $108.\n(integer) 108", "latencyMs": 11098 } }, { "before": { "text": "(9 dollars/hour * 7 hours on Saturday) + (9 dollars/hour * 5 hours on Sunday)\n= (9 * 7) + (9 * 5)\n= 63 + 45\n= 108\n<108>", "latencyMs": 6501 }, "after": { "text": "(9 dollars/hour * 7 hours on Saturday) + (9 dollars/hour * 5 hours on Sunday) = 63 + 45 = 108 dollars\n<108>", "latencyMs": 5054 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(9 dollars/hour * 7 hours for Saturday) + (9 dollars/hour * 5 hours for Sunday) = 63 + 45 = 108 dollars\n\n<108>", "latencyMs": 7121 }, "after": { "text": "(9 dollars/hour * 7 hours for Saturday + 9 dollars/hour * 5 hours for Sunday) \n(9 * 7 + 9 * 5) \n(63 + 45) \n108 \n<108>", "latencyMs": 7231 } }, { "before": { "text": "(9 dollars/hour * 7 hours on Saturday) + (9 dollars/hour * 5 hours on Sunday)\n= (9 * 7) + (9 * 5)\n= 63 + 45\n= 108\n<108>", "latencyMs": 6501 }, "after": { "text": "(9 dollars/hour * 7 hours + 9 dollars/hour * 5 hours) = 90 dollars \n \n90", "latencyMs": 3870 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 0.75, "equivalentCount": 1 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(9 dollars/hour * 7 hours for Saturday) + (9 dollars/hour * 5 hours for Sunday) = 63 + 45 = 108 dollars\n\n<108>", "latencyMs": 7121 }, "after": { "text": "(brief work)\n9 dollars/hour * 7 hours = 63 dollars for Saturday\n9 dollars/hour * 5 hours = 45 dollars for Sunday\nTotal = 63 dollars + 45 dollars = 108 dollars\n\n\n108", "latencyMs": 9819 } }, { "before": { "text": "(9 dollars/hour * 7 hours on Saturday) + (9 dollars/hour * 5 hours on Sunday)\n= (9 * 7) + (9 * 5)\n= 63 + 45\n= 108\n<108>", "latencyMs": 6501 }, "after": { "text": "(brief work)\nHannah earns $9 per hour.\nOn Saturday, she works for 7 hours, so she earns \\(7 \\times 9 = 63\\) dollars.\nOn Sunday, she works for 5 hours, so she earns \\(5 \\times 9 = 45\\) dollars.\nIn total for the weekend, she earns \\(63 + 45 = 108\\) dollars.\n\n108", "latencyMs": 10658 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a11", "expected": 50, "difficulty": "medium", "originalText": "A parking lot has 6 rows and each row has 15 parking spaces. If 40 cars are parked, how many empty spaces are there?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "Brief work:\n- Total parking spaces = Number of rows × Number of spaces per row\n- Total parking spaces = 6 × 15 = 90\n- Empty spaces = Total parking spaces - Number of cars parked\n- Empty spaces = 90 - 40 = 50\n\nAnswer: 50", "latencyMs": 8532 }, "after": { "text": "Brief work:\nTotal number of parking spaces = Number of rows * Number of spaces per row\nTotal number of parking spaces = 6 * 15 = 90\n\nNumber of empty spaces = Total number of parking spaces - Number of parked cars\nNumber of empty spaces = 90 - 40 = 50\n\nAnswer: <50>", "latencyMs": 7643 } }, { "before": { "text": "Brief work:\nTotal parking spaces = Number of rows * Spaces per row\nTotal parking spaces = 6 * 15\nTotal parking spaces = 90\n\nNumber of empty spaces = Total parking spaces - Number of parked cars\nNumber of empty spaces = 90 - 40\nNumber of empty spaces = 50\n\nAnswer: <50>", "latencyMs": 8007 }, "after": { "text": "Brief work:\nTotal parking spaces = Number of rows * Spaces per row\nTotal parking spaces = 6 * 15\nTotal parking spaces = 90\n\nNumber of empty spaces = Total parking spaces - Number of parked cars\nNumber of empty spaces = 90 - 40\nNumber of empty spaces = 50\n\nAnswer: <50>", "latencyMs": 7926 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "Brief work:\n- Total parking spaces = Number of rows × Number of spaces per row\n- Total parking spaces = 6 × 15 = 90\n- Empty spaces = Total parking spaces - Number of cars parked\n- Empty spaces = 90 - 40 = 50\n\nAnswer: 50", "latencyMs": 8532 }, "after": { "text": "(Each row has 15 spaces, so with 6 rows there are 15 * 6 = 90 spaces in total. If 40 cars are parked, the number of empty spaces is 90 - 40 = 50.)\n<50>", "latencyMs": 8628 } }, { "before": { "text": "Brief work:\nTotal parking spaces = Number of rows * Spaces per row\nTotal parking spaces = 6 * 15\nTotal parking spaces = 90\n\nNumber of empty spaces = Total parking spaces - Number of parked cars\nNumber of empty spaces = 90 - 40\nNumber of empty spaces = 50\n\nAnswer: <50>", "latencyMs": 8007 }, "after": { "text": "(6 rows * 15 spaces per row) - 40 parked cars = total spaces - parked cars\n(6 * 15) - 40 = 90 - 40 = 50\n\n<50>", "latencyMs": 5706 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "Brief work:\n- Total parking spaces = Number of rows × Number of spaces per row\n- Total parking spaces = 6 × 15 = 90\n- Empty spaces = Total parking spaces - Number of cars parked\n- Empty spaces = 90 - 40 = 50\n\nAnswer: 50", "latencyMs": 8532 }, "after": { "text": "Brief work:\nTotal parking spaces = Number of rows × Spaces per row = 6 × 15 = 90 spaces\n\nCars parked = 40\n\nEmpty spaces = Total parking spaces - Cars parked = 90 - 40 = 50\n\nAnswer: <50>", "latencyMs": 7708 } }, { "before": { "text": "Brief work:\nTotal parking spaces = Number of rows * Spaces per row\nTotal parking spaces = 6 * 15\nTotal parking spaces = 90\n\nNumber of empty spaces = Total parking spaces - Number of parked cars\nNumber of empty spaces = 90 - 40\nNumber of empty spaces = 50\n\nAnswer: <50>", "latencyMs": 8007 }, "after": { "text": "Brief work:\nTotal parking spaces = Number of rows * Spaces per row\nTotal parking spaces = 6 * 15\nTotal parking spaces = 90\n\nTotal empty spaces = Total parking spaces - Number of parked cars\nTotal empty spaces = 90 - 40\nTotal empty spaces = 50\n\nAnswer: <50>", "latencyMs": 10065 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] }, { "problemId": "a12", "expected": 56, "difficulty": "medium", "originalText": "Ian collected 84 seashells at the beach. He gave one third of them to his younger brother. How many seashells does Ian have left?", "aggregate": { "verdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "reason": "invariant_holds_across_all", "trueCount": 3, "falseCount": 0, "neitherCount": 0, "zeroCount": 0 }, "transforms": [ { "transformId": "variable-rename", "pairs": [ { "before": { "text": "(brief work)\nIan gave away \\( \\frac{84}{3} = 28 \\) seashells.\nSo, Ian has \\( 84 - 28 = 56 \\) seashells left.\n\n<56>", "latencyMs": 7130 }, "after": { "text": "(brief work)\nRui gave away one third of 84 seashells to his younger brother. One third of 84 is calculated as \\( \\frac{84}{3} = 28 \\).\nSo, Rui gave away 28 seashells.\n\nTo find out how many seashells Rui has left, we subtract the number of seashells given away from the total: \\( 84 - 28 = 56 \\).\n\n(final line)\n<56>", "latencyMs": 11971 } }, { "before": { "text": "(brief work)\nIan gave away \\( \\frac{1}{3} \\) of 84 seashells, which is \\( \\frac{84}{3} = 28 \\) seashells.\nTherefore, Ian has \\( 84 - 28 = 56 \\) seashells left.\n(integer) 56", "latencyMs": 7432 }, "after": { "text": "(brief work)\nRui gave away 84 / 3 = 28 seashells to his younger brother.\nSo, Rui has 84 - 28 = 56 seashells left.\n\n(integer) 56", "latencyMs": 5644 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "whitespace-noise", "pairs": [ { "before": { "text": "(brief work)\nIan gave away \\( \\frac{84}{3} = 28 \\) seashells.\nSo, Ian has \\( 84 - 28 = 56 \\) seashells left.\n\n<56>", "latencyMs": 7130 }, "after": { "text": "(brief work)\nIan gave away \\( \\frac{84}{3} = 28 \\) seashells.\nSo, he has \\( 84 - 28 = 56 \\) seashells left.\n\n(integer) 56", "latencyMs": 6629 } }, { "before": { "text": "(brief work)\nIan gave away \\( \\frac{1}{3} \\) of 84 seashells, which is \\( \\frac{84}{3} = 28 \\) seashells.\nTherefore, Ian has \\( 84 - 28 = 56 \\) seashells left.\n(integer) 56", "latencyMs": 7432 }, "after": { "text": "(brief work)\nIan gave away \\( \\frac{1}{3} \\) of 84 seashells, which is \\( \\frac{84}{3} = 28 \\) seashells.\nTherefore, Ian has \\( 84 - 28 = 56 \\) seashells left.\n(integer) 56", "latencyMs": 7865 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 }, { "transformId": "synonym-swap", "pairs": [ { "before": { "text": "(brief work)\nIan gave away \\( \\frac{84}{3} = 28 \\) seashells.\nSo, Ian has \\( 84 - 28 = 56 \\) seashells left.\n\n<56>", "latencyMs": 7130 }, "after": { "text": "(brief work)\nIan handed over \\( \\frac{84}{3} = 28 \\) shells to his younger brother.\nSo, Ian has \\( 84 - 28 = 56 \\) shells left.\n\n(integer) 56", "latencyMs": 6854 } }, { "before": { "text": "(brief work)\nIan gave away \\( \\frac{1}{3} \\) of 84 seashells, which is \\( \\frac{84}{3} = 28 \\) seashells.\nTherefore, Ian has \\( 84 - 28 = 56 \\) seashells left.\n(integer) 56", "latencyMs": 7432 }, "after": { "text": "(brief work)\nIan handed over \\( \\frac{1}{3} \\) of 84 shells to his younger brother, which is \\( \\frac{84}{3} = 28 \\) shells.\nTherefore, Ian has \\( 84 - 28 = 56 \\) shells left.\n\n\n56", "latencyMs": 8743 } } ], "singleVerdict": { "name": "TRUE", "symbol": "⊤", "numeric": 1 }, "singleReason": "invariant_preserved:numeric_match(diff=0.000e+0)", "statisticalVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "statisticalReason": "no_naive_signal", "pValue": 1, "equivalentCount": 2 } ] } ], "crossFdr": { "aggregateVerdict": { "name": "FALSE", "symbol": "⊥", "numeric": 0 }, "aggregateReason": "no_naive_signal", "trialCount": 36, "naiveSignificantCount": 0, "fdrSurvivorCount": 0 }, "summary": { "problemCount": 12, "transformCount": 3, "aggregateVerdictCounts": { "TRUE": 11, "FALSE": 0, "BOTH": 1, "NEITHER": 0, "ZERO": 0 } } }