#!/usr/bin/env python3 # ForgeBench 2026.10 scoring script. # # Copyright (c) 2026 ForgeAI. License: MIT is proposed but not yet signed off, so # no license is granted yet. You may run this script to check the published # ForgeBench results; ask ForgeAI before republishing or redistributing it. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. """Re-score the ForgeBench 2026.10 Official dungeon leaderboard from its run records. This is the scoring step only, not the harness or the game engine. It reads the published run records (the JSON or the CSV from https://forgeai.gg/api/forgebench/suite/2026.10/runs.json or .csv) and computes, per model, what the leaderboard on https://forgeai.gg/forgebench shows: Quality mean over the scored runs of 100 x (score - low) / (high - low), floored at 0 and not capped above 100 (each dungeon's frozen anchors) interval the 95% bootstrap percentile interval of Quality: 2000 resamples of the scored runs, from the same seeded generator the app uses, so the bounds reproduce exactly position one more than the number of models whose interval lies entirely above this one's ("=N" when shared); only models at the sample floor Completion solved runs / scored runs Cost per run mean dollars over the scored runs and the model's own voids that report a cost Cost per finish the same spend / solved runs Adherence 1 - failed decisions / decisions, pooled over the scored runs Depth median progress reached, percent of the dungeon Survival share of scored runs in which the agent did not die Python 3.8 or later, standard library only. python3 score_2026_10.py download the live JSON and print the table python3 score_2026_10.py runs.json read a saved JSON file (or runs.csv) python3 score_2026_10.py --json FILE print the computed rows as JSON python3 score_2026_10.py --check [FILE] recompute and compare with the leaderboard the JSON carries (default: the fixture snapshot next to this script, or downloaded) """ import argparse import csv import io import json import math import os import sys import urllib.request LIVE_URL = "https://forgeai.gg/api/forgebench/suite/2026.10/runs.json" FIXTURE_URL = "https://forgeai.gg/forgebench/open-data/fixture-2026.10-runs.json" FIXTURE_FILE = "fixture-2026.10-runs.json" DATA_SCHEMA = "forgebench.suite-open-data.v1" # The 2026.10 method, as the JSON's `method` block states it. The JSON's values # win; these are used for a CSV, which carries rows only. DEFAULT_METHOD = { "scoreFloor": 0, "scoreScale": 100, "sampleFloor": 10, "plannedRunsPerModel": 10, "intervalLevel": 0.95, "resamples": 2000, "bootstrapSeed": "forgebench-suite-score", "subscriptionRoutes": ["claude-code"], "stopBeforeCallReasons": ["run-cost-cap", "model-cost-cap"], } DEFAULT_SCENARIO_ID = "dungeons-frozen-10" MODEL_EXECUTORS = ("direct", "model+jev") U32 = 0xFFFFFFFF # -------------------------------------------------------------------------- # Reading the data # -------------------------------------------------------------------------- CSV_FIELDS = { "run_id": ("runId", "text"), "model_id": ("modelId", "text"), "model_label": ("modelLabel", "text"), "executor": ("executor", "text"), "route": ("route", "text"), "scaffold": ("scaffold", "text"), "prompt_version": ("promptVersion", "text"), "dungeon_id": ("dungeonId", "text"), "repeat": ("repeat", "int"), "tier": ("tier", "text"), "anchor_low": ("anchorLow", "number"), "anchor_high": ("anchorHigh", "number"), "status": ("status", "text"), "invalid_reason": ("invalidReason", "text"), "invalid_cause": ("invalidCause", "text"), "superseded": ("superseded", "bool"), "superseded_by": ("supersededBy", "text"), "score": ("score", "number"), "success": ("success", "bool"), "depth": ("depth", "number"), "survived": ("survived", "bool"), "decisions_total": ("decisionsTotal", "int"), "decisions_failed": ("decisionsFailed", "int"), "cost_usd": ("costUsd", "number"), "cost_source": ("costSource", "text"), "started_at": ("startedAt", "text"), "finished_at": ("finishedAt", "text"), "manifest_hash": ("manifestHash", "text"), "record_hash": ("recordHash", "text"), } def _csv_value(text, kind): if text == "": return None if kind == "bool": return text == "true" if kind == "int": return int(text) if kind == "number": value = float(text) return int(value) if value.is_integer() and "." not in text and "e" not in text else value # The export prefixes a text cell that starts like a spreadsheet formula with '. if len(text) > 1 and text[0] == "'" and text[1] in "=+-@\t\r": return text[1:] return text def read_source(source): """The text of a path or an http(s) URL.""" if source.startswith("http://") or source.startswith("https://"): request = urllib.request.Request(source, headers={"User-Agent": "forgebench-score/1"}) with urllib.request.urlopen(request, timeout=60) as response: return response.read().decode("utf-8") with open(source, encoding="utf-8") as handle: return handle.read() def load_dataset(source): """A dataset dict in the JSON's shape, from the JSON or the CSV.""" text = read_source(source) if text.lstrip().startswith("{"): data = json.loads(text) if data.get("schema") != DATA_SCHEMA: raise SystemExit("Unexpected schema %r; this script reads %s." % (data.get("schema"), DATA_SCHEMA)) return data rows = [] for record in csv.DictReader(io.StringIO(text)): rows.append({CSV_FIELDS[key][0]: _csv_value(value, CSV_FIELDS[key][1]) for key, value in record.items()}) anchors = {} for row in rows: if row["dungeonId"] is not None and row["anchorLow"] is not None: anchors[row["dungeonId"]] = { "dungeonId": row["dungeonId"], "tier": row["tier"], "low": row["anchorLow"], "high": row["anchorHigh"], } models = {} for row in rows: if row["executor"] in MODEL_EXECUTORS and row["modelId"] not in models: models[row["modelId"]] = {"id": row["modelId"], "label": row["modelLabel"], "route": row["route"]} return { "schema": DATA_SCHEMA, "scenario": {"id": DEFAULT_SCENARIO_ID, "dungeons": len(anchors)}, "method": dict(DEFAULT_METHOD), "models": sorted(models.values(), key=lambda model: model["id"]), "anchors": sorted(anchors.values(), key=lambda anchor: anchor["dungeonId"]), "rows": rows, "rejected": [], "leaderboard": None, } # -------------------------------------------------------------------------- # Arithmetic, in the same order of operations as the app's score module, so # every figure is the same double. # -------------------------------------------------------------------------- def mean(values): if not values: return None total = 0.0 for value in values: # a plain loop: Python 3.12's sum() of floats is compensated total += value return total / len(values) def median(values): if not values: return None ordered = sorted(values) middle = len(ordered) // 2 if len(ordered) % 2 == 1: return ordered[middle] return (ordered[middle - 1] + ordered[middle]) / 2 def quantile(ordered, p): """Linear-interpolation quantile of a sorted list.""" position = (len(ordered) - 1) * p below = math.floor(position) above = math.ceil(position) return ordered[below] + (ordered[above] - ordered[below]) * (position - below) def percentile_interval(values, level): tail = (1 - level) / 2 ordered = sorted(values) return {"low": quantile(ordered, tail), "high": quantile(ordered, 1 - tail)} def hash_seed(text): """FNV-1a over UTF-16 code units, as 32-bit unsigned.""" value = 0x811C9DC5 data = text.encode("utf-16-le") for index in range(0, len(data), 2): value ^= data[index] | (data[index + 1] << 8) value = (value * 0x01000193) & U32 return value def seeded_random(seed): """mulberry32, seeded with FNV-1a of the seed text: the app's generator, bit for bit.""" state = [hash_seed(seed)] def imul(left, right): return (left * right) & U32 def next_value(): state[0] = (state[0] + 0x6D2B79F5) & U32 current = state[0] mixed = imul(current ^ (current >> 15), 1 | current) mixed = ((mixed + imul(mixed ^ (mixed >> 7), 61 | mixed)) & U32) ^ mixed return ((mixed ^ (mixed >> 14)) & U32) / 4294967296 return next_value # -------------------------------------------------------------------------- # The leaderboard # -------------------------------------------------------------------------- def group_key(row): # The dungeon scenario has no arms or levels and pools its units: one group. return "arm=-|level=-" def scenario_quality(runs): """Mean within each sample group, then over groups.""" groups = {} for run in runs: groups.setdefault(run["group"], []).append(run["perf"]) return mean([mean(values) for values in groups.values()]) def quality_interval(runs, model_id, scenario_id, method): random = seeded_random("%s|%s|%s|quality" % (method["bootstrapSeed"], model_id, scenario_id)) plan = [] groups = {} for run in runs: groups.setdefault(run["group"], []).append(run) for key in sorted(groups): clusters = {} for run in groups[key]: clusters.setdefault(run["cluster"], []).append(run) plan.append([clusters[cluster] for cluster in sorted(clusters)]) draws = [] for _ in range(method["resamples"]): drawn = [] for clusters in plan: for _draw in range(len(clusters)): drawn.extend(clusters[math.floor(random() * len(clusters))]) draws.append(scenario_quality(drawn)) return percentile_interval(draws, method["intervalLevel"]) class Refused(Exception): pass def score(dataset): method = dict(DEFAULT_METHOD) method.update(dataset.get("method") or {}) scenario_id = (dataset.get("scenario") or {}).get("id", DEFAULT_SCENARIO_ID) anchors = {anchor["dungeonId"]: anchor for anchor in dataset["anchors"]} listed = {model["id"]: model for model in dataset["models"]} rows = [row for row in dataset["rows"] if not row["superseded"]] model_rows = [row for row in rows if row["executor"] in MODEL_EXECUTORS and row["modelId"] in listed] refusals = [] if dataset.get("rejected"): refusals.append("%d stored rows were rejected by the read path" % len(dataset["rejected"])) for row in model_rows: if row["status"] == "invalid" and row["invalidCause"] is None: refusals.append("invalid reason %r of run %s is not classified" % (row["invalidReason"], row["runId"])) cells = {} for row in model_rows: if row["status"] == "scored": key = (row["modelId"], row["dungeonId"], row["repeat"]) cells[key] = cells.get(key, 0) + 1 for key, count in sorted(cells.items(), key=lambda item: str(item[0])): if count > 1: refusals.append("%s has %d scored runs in one cell (%s)" % (key[0], count, key[1])) if refusals: raise Refused(refusals) out = [] for model in dataset["models"]: model_id = model["id"] mine = [row for row in model_rows if row["modelId"] == model_id] scored = sorted((row for row in mine if row["status"] == "scored"), key=lambda row: row["runId"]) invalid = [row for row in mine if row["status"] == "invalid"] charged = sorted((row for row in invalid if row["invalidCause"] == "model"), key=lambda row: row["runId"]) if len(scored) > method["plannedRunsPerModel"]: raise Refused(["%s has more scored runs than the plan" % model_id]) anchored = len(scored) > 0 for row in scored: anchor = anchors.get(row["dungeonId"]) if anchor is None or anchor["high"] <= anchor["low"] or row["score"] is None: anchored = False break runs = [] if anchored: for row in scored: anchor = anchors[row["dungeonId"]] perf = (method["scoreScale"] * (float(row["score"]) - float(anchor["low"]))) / ( float(anchor["high"]) - float(anchor["low"]) ) runs.append({"perf": max(float(method["scoreFloor"]), perf), "group": group_key(row), "cluster": row["runId"]}) quality = scenario_quality(runs) if anchored else None below_floor = len(scored) < method["sampleFloor"] interval = None if quality is not None and not below_floor: interval = quality_interval(runs, model_id, scenario_id, method) finished = sum(1 for row in scored if row["success"] is True) costs = [row["costUsd"] for row in scored + charged if row["costUsd"] is not None] spent = 0 for cost in costs: spent = spent + cost total = sum(row["decisionsTotal"] or 0 for row in scored) failed = sum(row["decisionsFailed"] or 0 for row in scored) adherence = None if total == 0 else (method["scoreScale"] * (1 - failed / total)) / 100 depths = [row["depth"] for row in scored if row["depth"] is not None] survival = [row["survived"] for row in scored if row["survived"] is not None] partial = False partly_estimated = False for row in scored: partial = partial or row["costSource"] == "openrouter_usage_partial" partly_estimated = partly_estimated or row["costSource"] == "openrouter_usage_with_list_estimate" for row in charged: unknown = row["costUsd"] is None and row["invalidReason"] not in method["stopBeforeCallReasons"] partial = partial or unknown or row["costSource"] == "openrouter_usage_partial" partly_estimated = partly_estimated or row["costSource"] == "openrouter_usage_with_list_estimate" out.append( { "modelId": model_id, "label": model["label"], "route": model["route"], "quality": quality, "qualityInterval": interval, "position": None, "completion": finished / len(scored) if scored else None, "finished": finished, "scored": len(scored), "planned": method["plannedRunsPerModel"], "dungeonsPlayed": len({row["dungeonId"] for row in scored if row["dungeonId"] is not None}), "dungeonsTotal": (dataset.get("scenario") or {}).get("dungeons", len(anchors)), "belowSampleFloor": below_floor, "invalid": len(invalid), "chargedInvalid": len(charged), "costPerRun": spent / len(costs) if costs else None, "costPerFinish": None if finished == 0 or not costs else spent / finished, "costEstimated": model["route"] in method["subscriptionRoutes"], "costPartial": partial, "costPartlyEstimated": partly_estimated, "adherence": adherence, "depth": median(depths), "survival": (sum(1 for value in survival if value) / len(survival)) if survival else None, } ) ranked = [row for row in out if row["quality"] is not None and row["qualityInterval"] is not None and not row["belowSampleFloor"]] ranks = {} for row in ranked: ranks[row["modelId"]] = 1 + sum(1 for other in ranked if other["qualityInterval"]["low"] > row["qualityInterval"]["high"]) for row in out: if row["modelId"] in ranks: rank = ranks[row["modelId"]] tied = sum(1 for value in ranks.values() if value == rank) > 1 row["position"] = {"rank": rank, "tied": tied, "label": ("=%d" % rank) if tied else str(rank)} def order(row): position = row["position"]["rank"] if row["position"] else math.inf quality = -row["quality"] if row["quality"] is not None else math.inf return (row["belowSampleFloor"], position, quality, row["label"].lower(), row["label"], row["modelId"]) return sorted(out, key=order) def check_anchor_lows(dataset): """Each dungeon's low anchor against the median of the random player's five seeds.""" results = [] for anchor in dataset["anchors"]: seeds = {} for row in dataset["rows"]: if ( row["modelId"] == "baseline:random" and row["status"] == "scored" and not row["superseded"] and row["dungeonId"] == anchor["dungeonId"] and row["score"] is not None ): seeds[row["repeat"]] = row["score"] if sorted(seeds) == [1, 2, 3, 4, 5]: results.append((anchor["dungeonId"], median(list(seeds.values())), anchor["low"])) return results # -------------------------------------------------------------------------- # Output # -------------------------------------------------------------------------- def _number(value, digits=1): return "-" if value is None else ("%.*f" % (digits, value)) def _percent(value): return "-" if value is None else "%.0f%%" % (value * 100) def _usd(value): if value is None: return "-" return "$%.2f" % value if value >= 1 else "$%.4f" % value def print_table(rows, method): header = ("Pos", "Model", "Quality", "95% interval", "Compl.", "Cost/run", "Cost/finish", "Adher.", "Depth", "Surv.", "Runs") lines = [header] for row in rows: interval = row["qualityInterval"] cost_mark = " est." if row["costEstimated"] else (" part est." if row["costPartlyEstimated"] else "") cost_mark += " (lower bound)" if row["costPartial"] else "" lines.append( ( row["position"]["label"] if row["position"] else "-", row["label"], _number(row["quality"]), "%s to %s" % (_number(interval["low"]), _number(interval["high"])) if interval else "-", _percent(row["completion"]), _usd(row["costPerRun"]) + cost_mark, _usd(row["costPerFinish"]), _percent(row["adherence"]), _number(row["depth"], 0), _percent(row["survival"]), "%d/%d%s" % (row["scored"], row["planned"], " insufficient" if row["belowSampleFloor"] else ""), ) ) widths = [max(len(str(line[index])) for line in lines) for index in range(len(header))] for line in lines: print(" ".join(str(cell).ljust(widths[index]) for index, cell in enumerate(line)).rstrip()) print() print( "Quality: 0 = median score of a random player, 100 = the dungeon's maximum. A position needs %d scored runs;" " positions are shared while 95%% intervals overlap." % method["sampleFloor"] ) COMPARED_FIELDS = ( "label", "route", "quality", "qualityInterval", "position", "completion", "finished", "scored", "planned", "dungeonsPlayed", "dungeonsTotal", "belowSampleFloor", "invalid", "chargedInvalid", "costPerRun", "costPerFinish", "costEstimated", "costPartial", "costPartlyEstimated", "adherence", "depth", "survival", ) def _compare(expected, actual, path, exact, close): if isinstance(expected, dict) and isinstance(actual, dict): for key in sorted(set(expected) | set(actual)): _compare(expected.get(key), actual.get(key), path + "." + key, exact, close) elif isinstance(expected, bool) or isinstance(actual, bool) or expected is None or actual is None: (exact if expected == actual and type(expected) is type(actual) else close).append((path, expected, actual, expected == actual)) elif isinstance(expected, (int, float)) and isinstance(actual, (int, float)): if float(expected) == float(actual): exact.append((path, expected, actual, True)) else: near = math.isclose(expected, actual, rel_tol=1e-9, abs_tol=1e-9) close.append((path, expected, actual, near)) else: (exact if expected == actual else close).append((path, expected, actual, expected == actual)) def run_check(dataset): board = dataset.get("leaderboard") if not board or board.get("status") != "ready": print("The file carries no ready leaderboard to check against: %s" % (board and board.get("reason"))) return 1 computed = score(dataset) by_id = {row["modelId"]: row for row in computed} exact, differ = [], [] order_ok = [row["modelId"] for row in board["rows"]] == [row["modelId"] for row in computed] for expected in board["rows"]: actual = by_id.get(expected["modelId"]) if actual is None: differ.append((expected["modelId"], "row", None, False)) continue for field in COMPARED_FIELDS: _compare(expected.get(field), actual.get(field), expected["modelId"] + "." + field, exact, differ) wrong = [entry for entry in differ if not entry[3]] for path, expected, actual, near in differ: print("%s %s: app %r, script %r" % ("ROUNDING" if near else "MISMATCH", path, expected, actual)) lows = check_anchor_lows(dataset) matching = sum(1 for _dungeon, value, low in lows if value == low) print("Rows: %d models, %d values compared, %d bit-identical, %d within 1e-9, %d different." % ( len(board["rows"]), len(exact) + len(differ), len(exact), len(differ) - len(wrong), len(wrong))) print("Order: %s." % ("same as the app's" if order_ok else "DIFFERENT from the app's")) if lows: print("Low anchors: %d of %d dungeons with all five random-player seeds match their median." % (matching, len(lows))) else: print("Low anchors: no dungeon in the file has all five random-player seeds.") if wrong or not order_ok or matching != len(lows): print("CHECK FAILED") return 1 print("CHECK PASSED" + ("" if not differ else " (some values differ in the last digits)")) return 0 def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) parser.add_argument("source", nargs="?", help="runs.json or runs.csv: a path or a URL (default: %s)" % LIVE_URL) parser.add_argument("--json", action="store_true", help="print the computed rows as JSON") parser.add_argument("--check", action="store_true", help="compare with the leaderboard the JSON carries") args = parser.parse_args(argv) source = args.source if source is None and args.check: local = os.path.join(os.path.dirname(os.path.abspath(__file__)), FIXTURE_FILE) source = local if os.path.exists(local) else FIXTURE_URL dataset = load_dataset(source or LIVE_URL) if args.check: return run_check(dataset) try: rows = score(dataset) except Refused as refused: print("Not scored: " + "; ".join(refused.args[0])) return 1 if args.json: print(json.dumps(rows, indent=2)) else: print_table(rows, dict(DEFAULT_METHOD, **(dataset.get("method") or {}))) return 0 if __name__ == "__main__": sys.exit(main())