{ "run": "postfix", "cases_version": "routegen-eval-v1", "cases_frozen": "2026-09-19", "meta": { "parse_calls_live": 34, "parse_calls_cached": 7, "parse_input_tokens": 82743, "parse_output_tokens": 3105, "parse_cost_usd": 0.098268, "wall_seconds": 668.4000000000001, "invocations": 3, "finished": "2026-09-20T11:03:26", "model": "claude-haiku-4-5" }, "cost": { "per_run": { "baseline": { "calls": 59, "usd": 0.1578 }, "postfix": { "calls": 34, "usd": 0.0983 }, "postfix (variability)": { "calls": 45, "usd": 0.133 }, "smoke": { "calls": 1, "usd": 0.0027 } }, "metered_usd": 0.39175499999999996, "metered_calls": 139, "distinct_parses": 120, "distinct_parses_usd": 0.3356, "campaign_total_usd": 0.44, "note": "Metered = what the harness recorded per run. Distinct parses = every unique request text ever sent, which also covers runs that were interrupted before writing a meta file. The campaign total adds the variability repeats back on top of the distinct-parse figure and is rounded up." }, "e2e": { "checks_passed": 50, "checks_failed": 0, "issues": 0, "stages": [ "HOSTILE / BOUNDARY INPUTS", "NEW USER: GENERATE A ROUTE", "EDIT CHAIN", "UPLOAD A FOREIGN GPX (lon-first attrs, rtept, extra junk)", "CANCEL + CONCURRENCY", "INTERVAL SPOT" ], "clean": true, "log": "evals/results/e2e_simulate.log" }, "overall": { "pass": 58, "partial": 1, "fail": 3, "inconclusive": 0, "total": 62 }, "by_split": { "dev": { "pass": 35, "partial": 1, "fail": 1, "inconclusive": 0, "total": 37 }, "heldout": { "pass": 23, "partial": 0, "fail": 2, "inconclusive": 0, "total": 25 } }, "by_tier": { "live": { "pass": 56, "partial": 1, "fail": 3, "inconclusive": 0, "total": 60 }, "offline": { "pass": 2, "partial": 0, "fail": 0, "inconclusive": 0, "total": 2 } }, "live_by_split": { "dev": { "pass": 33, "partial": 1, "fail": 1, "inconclusive": 0, "total": 35 }, "heldout": { "pass": 23, "partial": 0, "fail": 2, "inconclusive": 0, "total": 25 } }, "by_category": { "ambiguous": { "pass": 4, "partial": 0, "fail": 1, "inconclusive": 0, "total": 5 }, "avoid": { "pass": 3, "partial": 0, "fail": 0, "inconclusive": 0, "total": 3 }, "climb_max": { "pass": 3, "partial": 0, "fail": 0, "inconclusive": 0, "total": 3 }, "climb_min": { "pass": 5, "partial": 0, "fail": 0, "inconclusive": 0, "total": 5 }, "distance_target": { "pass": 5, "partial": 0, "fail": 0, "inconclusive": 0, "total": 5 }, "edit": { "pass": 15, "partial": 0, "fail": 2, "inconclusive": 0, "total": 17 }, "infeasible": { "pass": 3, "partial": 0, "fail": 0, "inconclusive": 0, "total": 3 }, "interval_spot": { "pass": 2, "partial": 1, "fail": 0, "inconclusive": 0, "total": 3 }, "multi_constraint": { "pass": 4, "partial": 0, "fail": 0, "inconclusive": 0, "total": 4 }, "out_of_scope": { "pass": 4, "partial": 0, "fail": 0, "inconclusive": 0, "total": 4 }, "provider_failure": { "pass": 2, "partial": 0, "fail": 0, "inconclusive": 0, "total": 2 }, "simple_loop": { "pass": 5, "partial": 0, "fail": 0, "inconclusive": 0, "total": 5 }, "via": { "pass": 3, "partial": 0, "fail": 0, "inconclusive": 0, "total": 3 } }, "checks": [ { "check": "parse.fields", "hard": true, "passed": 94, "failed": 2, "graded": 96, "info_only": 0, "pass_pct": 97.9 }, { "check": "outcome.no_crash", "hard": true, "passed": 62, "failed": 0, "graded": 62, "info_only": 0, "pass_pct": 100.0 }, { "check": "parse.request_type", "hard": true, "passed": 61, "failed": 1, "graded": 62, "info_only": 0, "pass_pct": 98.4 }, { "check": "geometry.continuous", "hard": true, "passed": 53, "failed": 0, "graded": 53, "info_only": 0, "pass_pct": 100.0 }, { "check": "gpx.valid", "hard": true, "passed": 53, "failed": 0, "graded": 53, "info_only": 0, "pass_pct": 100.0 }, { "check": "geometry.repeat", "hard": false, "passed": 47, "failed": 0, "graded": 47, "info_only": 6, "pass_pct": 100.0 }, { "check": "outcome.succeeded", "hard": true, "passed": 44, "failed": 2, "graded": 46, "info_only": 0, "pass_pct": 95.7 }, { "check": "route.distance", "hard": true, "passed": 39, "failed": 0, "graded": 39, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.starts_where_asked", "hard": true, "passed": 39, "failed": 0, "graded": 39, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.loop_closure", "hard": true, "passed": 19, "failed": 0, "graded": 19, "info_only": 0, "pass_pct": 100.0 }, { "check": "outcome.handled", "hard": false, "passed": 12, "failed": 0, "graded": 12, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.via[*]", "hard": true, "passed": 10, "failed": 0, "graded": 10, "info_only": 0, "pass_pct": 100.0 }, { "check": "outcome.explained", "hard": true, "passed": 7, "failed": 0, "graded": 7, "info_only": 0, "pass_pct": 100.0 }, { "check": "outcome.declined", "hard": true, "passed": 4, "failed": 0, "graded": 4, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.climb_cap", "hard": true, "passed": 4, "failed": 0, "graded": 4, "info_only": 0, "pass_pct": 100.0 }, { "check": "outcome.flagged_unsupported", "hard": true, "passed": 3, "failed": 0, "graded": 3, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.climb_sought", "hard": false, "passed": 3, "failed": 0, "graded": 3, "info_only": 0, "pass_pct": 100.0 }, { "check": "spot.long_enough_for_a_rep", "hard": false, "passed": 2, "failed": 1, "graded": 3, "info_only": 0, "pass_pct": 66.7 }, { "check": "spot.within_travel_budget", "hard": true, "passed": 3, "failed": 0, "graded": 3, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.avoid[*]", "hard": false, "passed": 2, "failed": 0, "graded": 2, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.distance_delta", "hard": true, "passed": 2, "failed": 0, "graded": 2, "info_only": 0, "pass_pct": 100.0 }, { "check": "outcome.message_names_cause", "hard": true, "passed": 1, "failed": 0, "graded": 1, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.anchored", "hard": true, "passed": 1, "failed": 0, "graded": 1, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.ends_where_asked", "hard": true, "passed": 1, "failed": 0, "graded": 1, "info_only": 0, "pass_pct": 100.0 }, { "check": "route.outback_returns", "hard": true, "passed": 1, "failed": 0, "graded": 1, "info_only": 0, "pass_pct": 100.0 }, { "check": "spot.lap_count_disclosed", "hard": true, "passed": 1, "failed": 0, "graded": 1, "info_only": 0, "pass_pct": 100.0 } ], "distance": { "n": 39, "median_pct": 1.8, "mean_pct": 2.4, "p90_pct": 6.0, "max_pct": 9.7, "within_5pct": 33, "within_10pct": 39, "within_15pct": 39 }, "distance_live": { "n": 39, "median_pct": 1.8, "mean_pct": 2.4, "p90_pct": 6.0, "max_pct": 9.7, "within_5pct": 33, "within_10pct": 39, "within_15pct": 39 }, "timing": { "n": 59, "median_s": 9.0, "mean_s": 20.6, "p90_s": 100.8, "max_s": 175.2, "total_s": 1214.8 }, "failure_groups": [ { "first_failed_check": "outcome.succeeded", "n": 2, "ids": [ "D21", "H24" ] }, { "first_failed_check": "spot.long_enough_for_a_rep", "n": 1, "ids": [ "D18" ] }, { "first_failed_check": "parse.request_type", "n": 1, "ids": [ "H23" ] } ], "tolerances": { "distance": { "default": 0.15, "why": "The product's own acceptance band: routes/scoring.py rejects any candidate more than 15% from the target, so 15% is the promise being tested, not a number chosen to flatter the result. Edits use 18% because an edit re-routes around fixed endpoints and has less freedom to hit a length exactly." }, "loop_closure_m": { "default": 250.0, "why": "routes/editing._is_loop uses 250 m. Wider than it looks: a loop that starts and ends on opposite sides of a divided road is closed for a rider." }, "start_proximity_m": { "default": 600.0, "why": "Geocoding a landmark returns a building centroid; the router snaps to the nearest routable way. 600 m absorbs both without hiding a route that starts in the wrong place." }, "via_m": { "default": 2000.0, "why": "BRouterProvider.VIA_NEAR_M -- the app's own definition of 'the route goes through there'. A town is an area, and forcing the geocoded centroid creates touch-and-retreat tendrils. Tightened per-case to 1500 m for a specific landmark." }, "avoid_on_road_m": { "default": 200.0, "why": "Measured as meters ridden within 28 m of the named road's real OSM centerline (the same measure routes/road_avoid.py uses to verify an avoid edit). Under 200 m is a crossing or a forced first/last block, not riding the road." }, "teleport": { "default": "gap > 800 m AND a bearing change of 60 deg or more", "why": "A first attempt used a plain 600 m gap threshold and flagged three perfectly good routes: Wisconsin's section-line grid gives dead-straight mile-long roads, which BRouter emits as a single segment. Requiring the long segment to ALSO turn away from the route's heading keeps the check able to catch a real splice without punishing straight rural geometry. Recorded here because the threshold was wrong before it was right." }, "repeat_frac": { "default": 0.25, "why": "routes/scoring.py rejects loops above 25% repeated road. Reported for every route; only enforced on loops, because an out-and-back is 100% repeated BY DESIGN and flagging that would be a scorer bug, not a product bug." }, "climb_cap_slack": { "default": 0.0, "why": "A stated ceiling is a hard promise: routes/scoring.py rejects anything above it, so the eval does too, with no slack. The elevation figure itself carries roughly +/-10% instrument spread (see PROVENANCE), which is reported alongside." } }, "provenance": { "geometry": "BRouter 1.7.10, self-hosted, profile fastbike-quiet, OSM extract tiles W90_N40 / W95_N40 (southern Wisconsin).", "elevation": "SRTM-derived elevations carried in BRouter's GeoJSON output. The app's climb figure is routes/elevation.track_ascent, a model calibrated against RideWithGPS and one barometric .fit file; the devlog records that instruments themselves disagree by 8-16%, so +/-10% is the attainable accuracy. The eval reports the app's number AND an unsmoothed independent sum over the same elevations. Neither is ground truth -- only a barometric ride is.", "geocoding": "OSM Nominatim, public endpoint, no key.", "road_geometry": "OSM Overpass API, way centerlines, used for the avoid-road measurement.", "not_measured": "Whether a route is SAFE or PLEASANT to ride. No check in this harness establishes that. Traffic volume, shoulder width, pavement quality, sightlines and seasonal conditions are outside what OSM tags and this eval can see. Those claims would need field riding." }, "cases": [ { "id": "D01", "split": "dev", "category": "simple_loop", "tier": "live", "scope": "supported", "text": "give me an easy 12 mile loop from Monona Terrace, Madison WI", "start": "Monona Terrace, Madison WI", "why": "The floor case: a short loop from a named public start.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 12.3 mi, 494 ft, 2% repeat", "notes": "User described this as 'easy', suggesting minimal climbing.", "seconds": 11.85, "measured": { "gpx_path": "evals/results/postfix/work/D01/route_12mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 858, "distance_mi": 12.24, "app_reported_mi": 12.3, "ascent_ft_app": 495.0, "ascent_ft_independent": 486.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 269.0, "duplicate_points": 0, "repeat_frac": 0.018, "mirror_frac": 0.015, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 12, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 12.3 mi, 494 ft, 2% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "12 vs [10, 14]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'loop' vs allowed ['loop', 'both']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 858 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "12.24 mi vs target 12 mi (+2.0%, tolerance +/-15%)", "value": 12.24, "limit": 12 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 269 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.018, "limit": 0.25 } ] }, { "id": "D02", "split": "dev", "category": "simple_loop", "tier": "live", "scope": "supported", "text": "20 mile loop please", "start": "Olin Park, Madison WI", "why": "Minimal phrasing; the start comes from the session's configured starting point.", "verdict": "pass", "fail_reasons": [], "summary": "6 route(s) \u2014 best: #1 loop: 20.2 mi, 765 ft, 1% repeat", "notes": "", "seconds": 8.19, "measured": { "gpx_path": "evals/results/postfix/work/D02/route_20mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1297, "distance_mi": 19.97, "app_reported_mi": 20.2, "ascent_ft_app": 766.0, "ascent_ft_independent": 718.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 32.6, "duplicate_points": 0, "repeat_frac": 0.014, "mirror_frac": 0.025, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 6 candidate(s): 6 route(s) \u2014 best: #1 loop: 20.2 mi, 765 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "20 vs [18, 22]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'loop' vs allowed ['loop', 'both']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1297 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "19.97 mi vs target 20 mi (+0.2%, tolerance +/-15%)", "value": 19.97, "limit": 20 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 1 m (limit 250 m)", "value": 1.3, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 33 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.014, "limit": 0.25 } ] }, { "id": "D03", "split": "dev", "category": "simple_loop", "tier": "live", "scope": "supported", "text": "I'd like to ride about thirty miles starting and finishing at the Wisconsin State Capitol in Madison", "start": "Wisconsin State Capitol, Madison WI", "why": "Spelled-out number, round-trip phrasing, explicit public start.", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 30.7 mi, 1112 ft, 0% repeat", "notes": "", "seconds": 13.85, "measured": { "gpx_path": "evals/results/postfix/work/D03/route_30mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1193, "distance_mi": 30.64, "app_reported_mi": 30.7, "ascent_ft_app": 1112.0, "ascent_ft_independent": 1127.0, "loop_closure_m": 0.0, "start_proximity_m": 0.1, "end_proximity_m": 0.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 687.7, "duplicate_points": 0, "repeat_frac": 0.004, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 30.7 mi, 1112 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [27, 33]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'loop' vs allowed ['loop', 'both']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1193 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "30.64 mi vs target 30 mi (+2.1%, tolerance +/-15%)", "value": 30.64, "limit": 30 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 688 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.004, "limit": 0.25 } ] }, { "id": "D04", "split": "dev", "category": "distance_target", "tier": "live", "scope": "supported", "text": "something around an hour and a half of riding from Olin Park, Madison WI", "start": "Olin Park, Madison WI", "why": "Duration instead of distance -- the prompt instructs the model to assume 16 mph.", "verdict": "pass", "fail_reasons": [], "summary": "10 route(s) \u2014 best: #1 outback: 24.0 mi, 964 ft, 0% repeat", "notes": "Assuming 16 mph average pace for 90 minutes yields approximately 24 miles.", "seconds": 19.04, "measured": { "gpx_path": "evals/results/postfix/work/D04/route_24mi_ride_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1241, "distance_mi": 23.93, "app_reported_mi": 24.0, "ascent_ft_app": 964.0, "ascent_ft_independent": 949.0, "loop_closure_m": 0.0, "start_proximity_m": 0.3, "end_proximity_m": 0.3, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 308.8, "duplicate_points": 0, "repeat_frac": 0.476, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 24, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 10 candidate(s): 10 route(s) \u2014 best: #1 outback: 24.0 mi, 964 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "24 vs [20, 28]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1241 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "23.93 mi vs target 24 mi (+0.3%, tolerance +/-15%)", "value": 23.93, "limit": 24 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 309 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.476, "limit": null } ] }, { "id": "D05", "split": "dev", "category": "distance_target", "tier": "live", "scope": "supported", "text": "out and back, roughly 25 miles, from the Memorial Union in Madison WI", "start": "Memorial Union, Madison WI", "why": "Out-and-back shape must survive the parse and reach generation.", "verdict": "pass", "fail_reasons": [], "summary": "6 route(s) \u2014 best: #1 outback: 24.9 mi, 817 ft, 0% repeat", "notes": "", "seconds": 6.06, "measured": { "gpx_path": "evals/results/postfix/work/D05/route_25mi_ride_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1123, "distance_mi": 24.85, "app_reported_mi": 24.9, "ascent_ft_app": 813.0, "ascent_ft_independent": 795.0, "loop_closure_m": 0.0, "start_proximity_m": 15.3, "end_proximity_m": 15.3, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 766.2, "duplicate_points": 0, "repeat_frac": 0.482, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "outback", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 6 candidate(s): 6 route(s) \u2014 best: #1 outback: 24.9 mi, 817 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'outback' vs allowed ['outback']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1123 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "24.85 mi vs target 25 mi (+0.6%, tolerance +/-15%)", "value": 24.85, "limit": 25 }, { "name": "route.outback_returns", "status": "pass", "hard": true, "detail": "out-and-back finishes 0 m from the start (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 15.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 766 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.482, "limit": null } ] }, { "id": "D06", "split": "dev", "category": "distance_target", "tier": "live", "scope": "supported", "text": "45 miles, out and back or a loop, whatever works, from Cross Plains WI", "start": "Cross Plains, WI", "why": "'Either shape is fine' must map to shape=both, which generates two spec families.", "verdict": "pass", "fail_reasons": [], "summary": "6 route(s) \u2014 best: #1 outback: 45.5 mi, 2361 ft, 0% repeat, 0.1 mi major", "notes": "", "seconds": 14.05, "measured": { "gpx_path": "evals/results/postfix/work/D06/route_45mi_ride_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1415, "distance_mi": 45.46, "app_reported_mi": 45.5, "ascent_ft_app": 2363.0, "ascent_ft_independent": 2468.0, "loop_closure_m": 0.0, "start_proximity_m": 17.2, "end_proximity_m": 17.2, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 458.5, "duplicate_points": 0, "repeat_frac": 0.479, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 45, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 6 candidate(s): 6 route(s) \u2014 best: #1 outback: 45.5 mi, 2361 ft, 0% repeat, 0.1 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "45 vs [42, 48]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'both' vs allowed ['both']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1415 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "45.46 mi vs target 45 mi (+1.0%, tolerance +/-15%)", "value": 45.46, "limit": 45 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 17 m from the geocoded start (limit 600 m)", "value": 17.2, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 458 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.479, "limit": null } ] }, { "id": "D07", "split": "dev", "category": "climb_max", "tier": "live", "scope": "supported", "text": "40 mile loop from Mount Horeb WI with as much climbing as you can find", "start": "Mount Horeb, WI", "why": "Climb maximization triggers the peak-scouting path (Overpass + elevation search).", "verdict": "pass", "fail_reasons": [], "summary": "7 route(s) \u2014 best: #1 loop: 39.0 mi, 2895 ft, 2% repeat, 0.4 mi major", "notes": "", "seconds": 25.37, "measured": { "gpx_path": "evals/results/postfix/work/D07/route_40mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1406, "distance_mi": 38.97, "app_reported_mi": 39.0, "ascent_ft_app": 2895.0, "ascent_ft_independent": 3010.0, "loop_closure_m": 0.0, "start_proximity_m": 3.5, "end_proximity_m": 3.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.0, "duplicate_points": 0, "repeat_frac": 0.021, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 40, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 7 candidate(s): 7 route(s) \u2014 best: #1 loop: 39.0 mi, 2895 ft, 2% repeat, 0.4 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "40 vs [37, 43]", "value": null, "limit": null }, { "name": "parse.maximize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1406 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "38.97 mi vs target 40 mi (+2.6%, tolerance +/-15%)", "value": 38.97, "limit": 40 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 4 m from the geocoded start (limit 600 m)", "value": 3.5, "limit": 600.0 }, { "name": "route.climb_sought", "status": "pass", "hard": false, "detail": "2895 ft on a climb-maximizing request (expected at least 1200 ft for this terrain)", "value": 2895.0, "limit": 1200 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 669 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.021, "limit": 0.25 } ] }, { "id": "D08", "split": "dev", "category": "climb_max", "tier": "live", "scope": "supported", "text": "give me the hilliest 25 miles you can from Verona WI", "start": "Verona, WI", "why": "Superlative phrasing ('hilliest') rather than the literal 'maximize climbing'.", "verdict": "pass", "fail_reasons": [], "summary": "19 route(s) \u2014 best: #1 loop: 24.6 mi, 1672 ft, 1% repeat", "notes": "", "seconds": 111.97, "measured": { "gpx_path": "evals/results/postfix/work/D08/route_25mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 783, "distance_mi": 24.55, "app_reported_mi": 24.6, "ascent_ft_app": 1672.0, "ascent_ft_independent": 1715.0, "loop_closure_m": 0.0, "start_proximity_m": 1.9, "end_proximity_m": 1.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 609.1, "duplicate_points": 0, "repeat_frac": 0.006, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 19 candidate(s): 19 route(s) \u2014 best: #1 loop: 24.6 mi, 1672 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.maximize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 783 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "24.55 mi vs target 25 mi (+1.8%, tolerance +/-15%)", "value": 24.55, "limit": 25 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 2 m from the geocoded start (limit 600 m)", "value": 1.9, "limit": 600.0 }, { "name": "route.climb_sought", "status": "pass", "hard": false, "detail": "1672 ft on a climb-maximizing request (expected at least 800 ft for this terrain)", "value": 1672.0, "limit": 800 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 609 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.006, "limit": 0.25 } ] }, { "id": "D09", "split": "dev", "category": "climb_min", "tier": "live", "scope": "supported", "text": "a 20 mile loop from Stoughton WI, as flat as possible please, my legs are wrecked", "start": "Stoughton, WI", "why": "Climb minimization with conversational noise around it.", "verdict": "pass", "fail_reasons": [], "summary": "6 route(s) \u2014 best: #1 loop: 19.0 mi, 540 ft, 1% repeat", "notes": "Rider prioritizes flattest terrain due to fatigue", "seconds": 3.05, "measured": { "gpx_path": "evals/results/postfix/work/D09/route_20mi_minclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1222, "distance_mi": 18.92, "app_reported_mi": 19.0, "ascent_ft_app": 533.0, "ascent_ft_independent": 469.0, "loop_closure_m": 0.0, "start_proximity_m": 1.1, "end_proximity_m": 1.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 29.8, "duplicate_points": 0, "repeat_frac": 0.007, "mirror_frac": 0.01, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 6 candidate(s): 6 route(s) \u2014 best: #1 loop: 19.0 mi, 540 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "20 vs [18, 22]", "value": null, "limit": null }, { "name": "parse.minimize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1222 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "18.92 mi vs target 20 mi (+5.4%, tolerance +/-15%)", "value": 18.92, "limit": 20 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 1 m from the geocoded start (limit 600 m)", "value": 1.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 30 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 1% -- not an out-and-back", "value": 0.007, "limit": 0.25 } ] }, { "id": "D10", "split": "dev", "category": "climb_min", "tier": "live", "scope": "supported", "text": "30 mile ride from Monona Terrace Madison WI with less than 1000 feet of climbing", "start": "Monona Terrace, Madison WI", "why": "A HARD climb ceiling -- the ranker must reject every candidate above it.", "verdict": "pass", "fail_reasons": [], "summary": "1 route(s) \u2014 best: #1 outback: 30.8 mi, 968 ft, 0% repeat", "notes": "", "seconds": 17.47, "measured": { "gpx_path": "evals/results/postfix/work/D10/route_30mi_minclimb_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1143, "distance_mi": 30.76, "app_reported_mi": 30.8, "ascent_ft_app": 961.0, "ascent_ft_independent": 963.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 766.2, "duplicate_points": 0, "repeat_frac": 0.479, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": 1000, "maximize_climb": false, "minimize_climb": true, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 1 candidate(s): 1 route(s) \u2014 best: #1 outback: 30.8 mi, 968 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [28, 32]", "value": null, "limit": null }, { "name": "parse.max_climb_ft", "status": "pass", "hard": true, "detail": "1000 vs [900, 1100]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1143 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "30.76 mi vs target 30 mi (+2.5%, tolerance +/-15%)", "value": 30.76, "limit": 30 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "route.climb_cap", "status": "pass", "hard": true, "detail": "961 ft vs cap 1000 ft (app model; independent unsmoothed sum 963 ft)", "value": 961.0, "limit": 1000 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 766 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.479, "limit": null } ] }, { "id": "D11", "split": "dev", "category": "climb_min", "tier": "live", "scope": "supported", "text": "not much climbing, about 25 miles, from Sun Prairie WI", "start": "Sun Prairie, WI", "why": "SOFT climbing language -- the prompt maps 'not much climbing' to roughly 1000 ft for rides up to 35 mi.", "verdict": "pass", "fail_reasons": [], "summary": "9 route(s) \u2014 best: #1 outback: 25.1 mi, 695 ft, 0% repeat", "notes": "", "seconds": 5.32, "measured": { "gpx_path": "evals/results/postfix/work/D11/route_25mi_ride_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1611, "distance_mi": 25.04, "app_reported_mi": 25.1, "ascent_ft_app": 695.0, "ascent_ft_independent": 677.0, "loop_closure_m": 0.0, "start_proximity_m": 3.4, "end_proximity_m": 3.4, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 301.6, "duplicate_points": 0, "repeat_frac": 0.484, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": 1000, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 9 candidate(s): 9 route(s) \u2014 best: #1 outback: 25.1 mi, 695 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.max_climb_ft", "status": "pass", "hard": true, "detail": "1000 vs [500, 1600]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1611 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "25.04 mi vs target 25 mi (+0.2%, tolerance +/-15%)", "value": 25.04, "limit": 25 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 3 m from the geocoded start (limit 600 m)", "value": 3.4, "limit": 600.0 }, { "name": "route.climb_cap", "status": "pass", "hard": true, "detail": "695 ft vs cap 1000 ft (app model; independent unsmoothed sum 677 ft)", "value": 695.0, "limit": 1000 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 302 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.484, "limit": null } ] }, { "id": "D12", "split": "dev", "category": "via", "tier": "live", "scope": "supported", "text": "a 25 mile loop from Monona Terrace Madison WI that goes through Olbrich Botanical Gardens", "start": "Monona Terrace, Madison WI", "why": "Single named waypoint on a loop.", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 23.5 mi, 916 ft, 2% repeat", "notes": "", "seconds": 14.92, "measured": { "gpx_path": "evals/results/postfix/work/D12/route_25mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1140, "distance_mi": 23.51, "app_reported_mi": 23.5, "ascent_ft_app": 919.0, "ascent_ft_independent": 887.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": { "Olbrich Botanical Gardens, Madison WI": 167.2 }, "avoid_on_road_m": {}, "max_point_gap_m": 766.2, "duplicate_points": 0, "repeat_frac": 0.019, "mirror_frac": 0.015, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [ "Olbrich Botanical Gardens, Madison, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 23.5 mi, 916 ft, 2% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1140 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "23.51 mi vs target 25 mi (+6.0%, tolerance +/-15%)", "value": 23.51, "limit": 25 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "route.via[Olbrich Botanical Gardens, Madison WI]", "status": "pass", "hard": true, "detail": "closest approach 167 m (limit 2000 m)", "value": 167.2, "limit": 2000 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 766 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.019, "limit": 0.25 } ] }, { "id": "D13", "split": "dev", "category": "via", "tier": "live", "scope": "supported", "text": "35 miles from Middleton WI, I want it to pass through Cross Plains and Black Earth", "start": "Middleton, WI", "why": "Two waypoints in ride order; both must be honored.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 34.0 mi, 1913 ft, 23% repeat, 0.1 mi major", "notes": "", "seconds": 9.7, "measured": { "gpx_path": "evals/results/postfix/work/D13/route_35mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1173, "distance_mi": 33.97, "app_reported_mi": 34.0, "ascent_ft_app": 1912.0, "ascent_ft_independent": 1900.0, "loop_closure_m": 0.0, "start_proximity_m": 22.9, "end_proximity_m": 22.9, "via_distances_m": { "Cross Plains, WI": 17.2, "Black Earth, WI": 2.8 }, "avoid_on_road_m": {}, "max_point_gap_m": 592.2, "duplicate_points": 0, "repeat_frac": 0.225, "mirror_frac": 0.45, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 35, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [ "Cross Plains, WI", "Black Earth, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 34.0 mi, 1913 ft, 23% repeat, 0.1 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "35 vs [32, 38]", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "2 item(s), need >= 2", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1173 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "33.97 mi vs target 35 mi (+2.9%, tolerance +/-15%)", "value": 33.97, "limit": 35 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 23 m from the geocoded start (limit 600 m)", "value": 22.9, "limit": 600.0 }, { "name": "route.via[Cross Plains, WI]", "status": "pass", "hard": true, "detail": "closest approach 17 m (limit 2500 m)", "value": 17.2, "limit": 2500 }, { "name": "route.via[Black Earth, WI]", "status": "pass", "hard": true, "detail": "closest approach 3 m (limit 2500 m)", "value": 2.8, "limit": 2500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 592 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "22% of the route rides the same road twice (the app rejects loops above 25%); mirror score 45%", "value": 0.225, "limit": 0.25 } ] }, { "id": "D14", "split": "dev", "category": "avoid", "tier": "live", "scope": "supported", "text": "30 mile loop from Monona Terrace Madison WI and keep me off Verona Road", "start": "Monona Terrace, Madison WI", "why": "Avoid-place on a fresh route (rides along as BRouter nogo circles).", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 29.8 mi, 1225 ft, 3% repeat", "notes": "", "seconds": 11.83, "measured": { "gpx_path": "evals/results/postfix/work/D14/route_30mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1427, "distance_mi": 29.77, "app_reported_mi": 29.8, "ascent_ft_app": 1224.0, "ascent_ft_independent": 1245.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 766.2, "duplicate_points": 0, "repeat_frac": 0.035, "mirror_frac": 0.05, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [ "Verona Road, Madison, WI" ], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 29.8 mi, 1225 ft, 3% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [28, 32]", "value": null, "limit": null }, { "name": "parse.avoid_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1427 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "29.77 mi vs target 30 mi (+0.8%, tolerance +/-15%)", "value": 29.77, "limit": 30 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 766 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "4% of the route rides the same road twice (the app rejects loops above 25%); mirror score 5% -- not an out-and-back", "value": 0.035, "limit": 0.25 } ] }, { "id": "D15", "split": "dev", "category": "avoid", "tier": "live", "scope": "supported", "text": "I hate riding on Whitney Way -- 22 mile loop from the Memorial Union Madison WI without it", "start": "Memorial Union, Madison WI", "why": "Dislike phrasing rather than the word 'avoid'.", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 22.0 mi, 881 ft, 1% repeat", "notes": "", "seconds": 10.55, "measured": { "gpx_path": "evals/results/postfix/work/D15/route_22mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1262, "distance_mi": 21.98, "app_reported_mi": 22.0, "ascent_ft_app": 881.0, "ascent_ft_independent": 899.0, "loop_closure_m": 0.0, "start_proximity_m": 15.3, "end_proximity_m": 15.3, "via_distances_m": {}, "avoid_on_road_m": { "Whitney Way, Madison WI": 0.0 }, "max_point_gap_m": 543.7, "duplicate_points": 0, "repeat_frac": 0.014, "mirror_frac": 0.02, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 22, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [ "Whitney Way, Madison, WI" ], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 22.0 mi, 881 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "22 vs [20, 24]", "value": null, "limit": null }, { "name": "parse.avoid_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1262 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "21.98 mi vs target 22 mi (+0.1%, tolerance +/-15%)", "value": 21.98, "limit": 22 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 15.3, "limit": 600.0 }, { "name": "route.avoid[Whitney Way, Madison WI]", "status": "pass", "hard": false, "detail": "0 m still ridden on it (limit 200 m)", "value": 0.0, "limit": 200.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 544 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.014, "limit": 0.25 } ] }, { "id": "D16", "split": "dev", "category": "multi_constraint", "tier": "live", "scope": "supported", "text": "40 mile loop from Verona WI, under 1500 ft of climbing, and stay off Highway 18", "start": "Verona, WI", "why": "Distance + hard climb cap + avoid, all at once.", "verdict": "pass", "fail_reasons": [], "summary": "1 route(s) \u2014 best: #1 loop: 37.1 mi, 1491 ft, 1% repeat", "notes": "", "seconds": 10.36, "measured": { "gpx_path": "evals/results/postfix/work/D16/route_40mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1131, "distance_mi": 37.03, "app_reported_mi": 37.1, "ascent_ft_app": 1492.0, "ascent_ft_independent": 1503.0, "loop_closure_m": 0.0, "start_proximity_m": 1.9, "end_proximity_m": 1.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 842.1, "duplicate_points": 0, "repeat_frac": 0.01, "mirror_frac": 0.015, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 40, "max_climb_ft": 1500, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [ "Highway 18, Verona, WI" ], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 1 candidate(s): 1 route(s) \u2014 best: #1 loop: 37.1 mi, 1491 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "40 vs [37, 43]", "value": null, "limit": null }, { "name": "parse.max_climb_ft", "status": "pass", "hard": true, "detail": "1500 vs [1400, 1600]", "value": null, "limit": null }, { "name": "parse.avoid_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1131 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "37.03 mi vs target 40 mi (+7.4%, tolerance +/-15%)", "value": 37.03, "limit": 40 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 2 m from the geocoded start (limit 600 m)", "value": 1.9, "limit": 600.0 }, { "name": "route.climb_cap", "status": "pass", "hard": true, "detail": "1492 ft vs cap 1500 ft (app model; independent unsmoothed sum 1503 ft)", "value": 1492.0, "limit": 1500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 842 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.01, "limit": 0.25 } ] }, { "id": "D17", "split": "dev", "category": "multi_constraint", "tier": "live", "scope": "supported", "text": "50 miles from Mount Horeb WI, loop, lots of climbing, and I want to go through Blue Mounds", "start": "Mount Horeb, WI", "why": "Climb maximization AND a waypoint -- in the code these two paths are mutually exclusive, so via wins and the climb goal becomes best-effort.", "verdict": "pass", "fail_reasons": [], "summary": "2 route(s) \u2014 best: #1 loop: 48.1 mi, 2500 ft, 0% repeat, 0.1 mi major", "notes": "", "seconds": 10.25, "measured": { "gpx_path": "evals/results/postfix/work/D17/route_50mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1277, "distance_mi": 48.09, "app_reported_mi": 48.1, "ascent_ft_app": 2500.0, "ascent_ft_independent": 2579.0, "loop_closure_m": 0.0, "start_proximity_m": 3.5, "end_proximity_m": 3.5, "via_distances_m": { "Blue Mounds, WI": 6.3 }, "avoid_on_road_m": {}, "max_point_gap_m": 688.6, "duplicate_points": 0, "repeat_frac": 0.003, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 50, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [ "Blue Mounds, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): 2 route(s) \u2014 best: #1 loop: 48.1 mi, 2500 ft, 0% repeat, 0.1 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "50 vs [47, 53]", "value": null, "limit": null }, { "name": "parse.maximize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1277 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "48.09 mi vs target 50 mi (+3.8%, tolerance +/-15%)", "value": 48.09, "limit": 50 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 4 m from the geocoded start (limit 600 m)", "value": 3.5, "limit": 600.0 }, { "name": "route.via[Blue Mounds, WI]", "status": "pass", "hard": true, "detail": "closest approach 6 m (limit 2500 m)", "value": 6.3, "limit": 2500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 689 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "D18", "split": "dev", "category": "interval_spot", "tier": "live", "scope": "supported", "text": "find me a flat stretch within 20 minutes of Olin Park Madison WI for 2x20 threshold intervals", "start": "Olin Park, Madison WI", "why": "The other request type entirely -- a stretch of road, not a route.", "verdict": "partial", "fail_reasons": [ "spot.long_enough_for_a_rep" ], "summary": "Found 3 interval spot(s) \u2014 best: #1: 5.6 mi @ +0.1%, 11 stops, 0.0 mi out \u2014 2 laps per rep (turnaround each lap). A 20-minute flat rep needs about 6.7 mi of road.", "notes": "", "seconds": 7.75, "measured": { "gpx_path": "evals/results/postfix/work/D18/spot_flat_2x20_1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 64, "distance_mi": 5.5, "app_reported_mi": 5.6, "ascent_ft_app": 122.0, "ascent_ft_independent": 127.0, "loop_closure_m": 6677.9, "start_proximity_m": 0.3, "end_proximity_m": 6677.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 766.2, "duplicate_points": 0, "repeat_frac": 0.0, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": null, "parsed_interval": { "kind": "flat", "max_travel_minutes": 20, "rep_minutes": 20, "reps": 2 }, "request_type": "interval_spot", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): Found 3 interval spot(s) \u2014 best: #1: 5.6 mi @ +0.1%, 11 stops, 0.0 mi out \u2014 2 laps per rep (turnaround each lap). A 20-minute flat rep needs about 6.7 mi of roa", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'interval_spot', expected 'interval_spot'", "value": "interval_spot", "limit": "interval_spot" }, { "name": "parse.reps", "status": "pass", "hard": true, "detail": "2 vs expected 2", "value": null, "limit": null }, { "name": "parse.rep_minutes", "status": "pass", "hard": true, "detail": "20 vs [18, 22]", "value": null, "limit": null }, { "name": "parse.kind", "status": "pass", "hard": true, "detail": "'flat' vs allowed ['flat']", "value": null, "limit": null }, { "name": "parse.max_travel_minutes", "status": "pass", "hard": true, "detail": "20 vs [15, 25]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 64 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 766 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.0, "limit": 0.25 }, { "name": "spot.within_travel_budget", "status": "pass", "hard": true, "detail": "stretch begins 0.0 km from the start; the stated budget of 20 min allows 8.0 km at the app's assumed 15 mph travel pace", "value": 0.0, "limit": 8047.0 }, { "name": "spot.long_enough_for_a_rep", "status": "fail", "hard": false, "detail": "5.50 mi of road for a 20-minute flat rep, which needs about 6.67 mi at the app's assumed pace (2 lap(s) per rep)", "value": 5.5, "limit": 6.67 }, { "name": "spot.lap_count_disclosed", "status": "pass", "hard": true, "detail": "the stretch needs 2 laps per rep; the text the rider sees says so: 'Found 3 interval spot(s) \u2014 best: #1: 5.6 mi @ +0.1%, 11 stops, 0.0 mi out \u2014 2 laps per rep (turnaround each lap). A 20-minute flat rep needs about 6.7 mi of road. #1: 5.6 mi @ +0.1%, 11 stops, 0.0 mi '", "value": null, "limit": null } ] }, { "id": "D19", "split": "dev", "category": "interval_spot", "tier": "live", "scope": "supported", "text": "I need a hill close to Verona WI for 4x5 minute VO2 efforts", "start": "Verona, WI", "why": "Incline kind inferred from 'VO2'; travel budget inferred from 'close to'.", "verdict": "pass", "fail_reasons": [], "summary": "Found 3 interval spot(s) \u2014 best: #1: 0.9 mi @ +3.3%, 0 stops, 4.2 mi out. A 5-minute incline rep needs about 0.9 mi of road.", "notes": "", "seconds": 100.77, "measured": { "gpx_path": "evals/results/postfix/work/D19/spot_incline_4x5_1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 11, "distance_mi": 0.91, "app_reported_mi": 0.9, "ascent_ft_app": 149.0, "ascent_ft_independent": 152.0, "loop_closure_m": 1447.9, "start_proximity_m": 5326.9, "end_proximity_m": 5998.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 317.4, "duplicate_points": 0, "repeat_frac": 0.0, "mirror_frac": 0.025, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": null, "parsed_interval": { "kind": "incline", "max_travel_minutes": 15, "rep_minutes": 5, "reps": 4 }, "request_type": "interval_spot", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): Found 3 interval spot(s) \u2014 best: #1: 0.9 mi @ +3.3%, 0 stops, 4.2 mi out. A 5-minute incline rep needs about 0.9 mi of road.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'interval_spot', expected 'interval_spot'", "value": "interval_spot", "limit": "interval_spot" }, { "name": "parse.reps", "status": "pass", "hard": true, "detail": "4 vs expected 4", "value": null, "limit": null }, { "name": "parse.rep_minutes", "status": "pass", "hard": true, "detail": "5 vs [4, 6]", "value": null, "limit": null }, { "name": "parse.kind", "status": "pass", "hard": true, "detail": "'incline' vs allowed ['incline']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 11 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 317 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.0, "limit": 0.25 }, { "name": "spot.within_travel_budget", "status": "pass", "hard": true, "detail": "stretch begins 5.3 km from the start; the stated budget of 15 min allows 6.0 km at the app's assumed 15 mph travel pace", "value": 5327.0, "limit": 6035.0 }, { "name": "spot.long_enough_for_a_rep", "status": "pass", "hard": false, "detail": "0.91 mi of road for a 5-minute incline rep, which needs about 0.92 mi at the app's assumed pace (1 lap(s) per rep)", "value": 0.91, "limit": 0.92 } ] }, { "id": "D20", "split": "dev", "category": "ambiguous", "tier": "live", "scope": "supported", "text": "I want to go for a bike ride", "start": "Wisconsin State Capitol, Madison WI", "why": "No distance, no shape, no constraint. The schema FORCES a distance, so the system must guess. Probes whether the guess is sane and whether the user is told it was a guess.", "verdict": "pass", "fail_reasons": [], "summary": "9 route(s) \u2014 best: #1 loop: 10.1 mi, 461 ft, 1% repeat, 0.3 mi major", "notes": "User did not specify a distance; defaulting to 10 miles. Please clarify desired length, climbing preference, or route shape (loop vs out-and-back).", "seconds": 15.37, "measured": { "gpx_path": "evals/results/postfix/work/D20/route_10mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 648, "distance_mi": 9.91, "app_reported_mi": 10.1, "ascent_ft_app": 461.0, "ascent_ft_independent": 488.0, "loop_closure_m": 0.0, "start_proximity_m": 0.1, "end_proximity_m": 0.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 25.1, "duplicate_points": 0, "repeat_frac": 0.014, "mirror_frac": 0.02, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 10, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 9 candidate(s): 9 route(s) \u2014 best: #1 loop: 10.1 mi, 461 ft, 1% repeat, 0.3 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "10 vs [5, 60]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 648 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "9.91 mi vs target 10 mi (+0.9%, tolerance +/-15%)", "value": 9.91, "limit": 10 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 25 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.014, "limit": 0.25 } ] }, { "id": "D21", "split": "dev", "category": "ambiguous", "tier": "live", "scope": "supported", "text": "a short easy spin from Olin Park Madison WI", "start": "Olin Park, Madison WI", "why": "Vague magnitude words only. 'Short' must not become 60 miles.", "verdict": "fail", "fail_reasons": [ "outcome.succeeded" ], "summary": "No route met the constraints \u2014 try a looser target or different distance.", "notes": "Short easy spin implies flat, leisurely ride around 10 miles; user emphasized easy and short so minimizing climb and keeping distance modest", "seconds": 8.69, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 10, "max_climb_ft": 300, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "fail", "hard": true, "detail": "ok=False, 0 candidate(s): No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "10 vs [4, 20]", "value": null, "limit": null } ] }, { "id": "D22", "split": "dev", "category": "ambiguous", "tier": "live", "scope": "supported", "text": "30 mile loop from Monona Terrace Madison WI, as flat as possible but also as much climbing as you can get", "start": "Monona Terrace, Madison WI", "why": "Directly contradictory. The system must not set BOTH maximize_climb and minimize_climb, and should say something about the conflict.", "verdict": "pass", "fail_reasons": [], "summary": "12 route(s) \u2014 best: #1 loop: 31.6 mi, 1800 ft, 4% repeat", "notes": "User requested both 'as flat as possible' AND 'as much climbing as you can get' \u2014 these are contradictory. Prioritizing maximize_climb=true as the more specific preference.", "seconds": 175.23, "measured": { "gpx_path": "evals/results/postfix/work/D22/route_30mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1719, "distance_mi": 31.56, "app_reported_mi": 31.6, "ascent_ft_app": 1799.0, "ascent_ft_independent": 1859.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 616.5, "duplicate_points": 0, "repeat_frac": 0.038, "mirror_frac": 0.03, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 12 candidate(s): 12 route(s) \u2014 best: #1 loop: 31.6 mi, 1800 ft, 4% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.not_both_climb_flags", "status": "pass", "hard": true, "detail": "maximize=True minimize=False", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1719 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "31.56 mi vs target 30 mi (+5.2%, tolerance +/-15%)", "value": 31.56, "limit": 30 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 616 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "4% of the route rides the same road twice (the app rejects loops above 25%); mirror score 3% -- not an out-and-back", "value": 0.038, "limit": 0.25 } ] }, { "id": "D23", "split": "dev", "category": "infeasible", "tier": "live", "scope": "supported", "text": "give me a 6 mile loop from Monona Terrace Madison WI with at least 5000 feet of climbing", "start": "Monona Terrace, Madison WI", "why": "Physically impossible in southern Wisconsin. The right answer is an honest refusal or a clearly-labelled best effort, not a route that pretends.", "verdict": "pass", "fail_reasons": [], "summary": "16 route(s) \u2014 best: #1 loop: 6.6 mi, 365 ft, 2% repeat", "notes": "User requested at least 5000 feet of climbing on a 6 mile loop; this is extremely steep terrain (average grade ~80%), which may not be achievable in most areas. Tool will maximize climbing within realistic constraints.", "seconds": 58.81, "measured": { "gpx_path": "evals/results/postfix/work/D23/route_6mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 561, "distance_mi": 6.58, "app_reported_mi": 6.6, "ascent_ft_app": 364.0, "ascent_ft_independent": 408.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 183.3, "duplicate_points": 0, "repeat_frac": 0.023, "mirror_frac": 0.035, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 6, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 16 candidate(s): 16 route(s) \u2014 best: #1 loop: 6.6 mi, 365 ft, 2% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "6 vs [5, 7]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 561 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "6.58 mi vs target 6 mi (+9.7%, tolerance +/-15%)", "value": 6.58, "limit": 6 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 183 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 4% -- not an out-and-back", "value": 0.023, "limit": 0.25 } ] }, { "id": "D24", "split": "dev", "category": "infeasible", "tier": "live", "scope": "supported", "text": "a 30 mile loop starting in Denver Colorado", "start": "Denver, CO", "why": "Outside the loaded routing tiles (southern Wisconsin). Probes the provider-has-no-data failure path.", "verdict": "pass", "fail_reasons": [], "summary": "No route met the constraints \u2014 try a looser target or different distance.", "notes": "", "seconds": 1.62, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.declined", "status": "pass", "hard": true, "detail": "ok=False, 0 candidate(s): No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [27, 33]", "value": null, "limit": null } ] }, { "id": "D25", "split": "dev", "category": "out_of_scope", "tier": "live", "scope": "out_of_scope", "text": "a 30 mile gravel loop from Cross Plains WI, mostly unpaved roads", "start": "Cross Plains, WI", "why": "Surface preference is NOT a supported constraint. Honest behavior is to flag it, not silently ignore it.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 30.4 mi, 1940 ft, 0% repeat, 0.2 mi major", "notes": "User requests mostly unpaved/gravel roads. Tool should prioritize gravel and dirt road segments.", "seconds": 4.76, "measured": { "gpx_path": "evals/results/postfix/work/D25/route_30mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 982, "distance_mi": 30.36, "app_reported_mi": 30.4, "ascent_ft_app": 1942.0, "ascent_ft_independent": 1976.0, "loop_closure_m": 0.0, "start_proximity_m": 17.2, "end_proximity_m": 17.2, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.0, "duplicate_points": 0, "repeat_frac": 0.002, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 30.4 mi, 1940 ft, 0% repeat, 0.2 mi major", "value": null, "limit": null }, { "name": "outcome.flagged_unsupported", "status": "pass", "hard": true, "detail": "unsupported part reported to the user: User requests mostly unpaved/gravel roads. Tool should prioritize gravel and dirt road segments.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.notes", "status": "pass", "hard": true, "detail": "present", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 982 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "30.36 mi vs target 30 mi (+1.2%, tolerance +/-15%)", "value": 30.36, "limit": 30 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 17 m from the geocoded start (limit 600 m)", "value": 17.2, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 669 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.002, "limit": 0.25 } ] }, { "id": "D26", "split": "dev", "category": "out_of_scope", "tier": "live", "scope": "out_of_scope", "text": "plan me a three day bikepacking trip from Madison WI with overnight stops", "start": "Wisconsin State Capitol, Madison WI", "why": "Multi-day touring is out of product scope entirely.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 outback: 151.0 mi, 6854 ft, 0% repeat", "notes": "User requested a three-day bikepacking trip with overnight stops. This is a multi-day adventure, not a single-day route. The tool generates single-day routes. Recommend: break this into three separate route requests (Day 1: Madison to stop 1, Day 2: stop 1 to stop 2, Day 3: stop 2 to destination), or specify the total distance and intermediate waypoints as via_places so the route passes through logical overnight locations. A typical three-day trip from Madison might total 150-200 miles depending on pace and terrain. User should clarify desired total distance, overnight destinations, and final destination.", "seconds": 9.04, "measured": { "gpx_path": "evals/results/postfix/work/D26/route_150mi_ride_1_outback_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 3309, "distance_mi": 150.81, "app_reported_mi": 151.0, "ascent_ft_app": 6858.0, "ascent_ft_independent": 7029.0, "loop_closure_m": 0.0, "start_proximity_m": 0.2, "end_proximity_m": 0.2, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1375.3, "duplicate_points": 2, "repeat_frac": 0.48, "mirror_frac": 1.0, "teleports": [], "gaps_confirmed_road": [ { "index": 1610, "gap_m": 802.1, "turn_in_deg": 88.6, "turn_out_deg": 89.0, "ratio": 1.0, "road": true, "road_m": 805.0, "straight_m": 802.1, "why": "road distance 805 m vs straight line 802 m (x1.00); a road runs along it" }, { "index": 1699, "gap_m": 802.1, "turn_in_deg": 89.0, "turn_out_deg": 88.6, "ratio": 1.0, "road": true, "road_m": 805.0, "straight_m": 802.1, "why": "road distance 805 m vs straight line 802 m (x1.00); a road runs along it" } ], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "avoid_places": [], "distance_miles": 150, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "outback", "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 outback: 151.0 mi, 6854 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.notes", "status": "pass", "hard": true, "detail": "present", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 3309 points, elevation present", "value": null, "limit": null }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.2, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1375 m; 2 long straight segment(s) were checked against the road network and a road runs along each", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "info", "hard": false, "detail": "48% repeated road, but the track is 100% its own reverse -- this is an out-and-back, where repetition is the shape, not a defect", "value": 0.48, "limit": null } ] }, { "id": "D27", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "a 22 mile loop from Olin Park Madison WI", "start": "Olin Park, Madison WI", "why": "Session E1 setup: the route the following edits chain against.", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 21.6 mi, 909 ft, 2% repeat", "notes": "", "seconds": 10.02, "measured": { "gpx_path": "evals/results/postfix/work/E1/route_22mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1390, "distance_mi": 21.4, "app_reported_mi": 21.6, "ascent_ft_app": 905.0, "ascent_ft_independent": 868.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 28.9, "duplicate_points": 0, "repeat_frac": 0.017, "mirror_frac": 0.025, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 22, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 21.6 mi, 909 ft, 2% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "22 vs [20, 24]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1390 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "21.40 mi vs target 22 mi (+2.7%, tolerance +/-15%)", "value": 21.4, "limit": 22 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 1 m (limit 250 m)", "value": 1.3, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 29 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.017, "limit": 0.25 } ] }, { "id": "D28", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "make it about 6 miles longer", "start": "Olin Park, Madison WI", "why": "Relative extend: a delta, not a total.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 extend 6mi: now 27.1 mi (+5.8 mi).", "notes": "", "seconds": 5.46, "measured": { "gpx_path": "evals/results/postfix/work/E1/route_22mi_ride_1_loop_brouter_edit1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1767, "distance_mi": 27.19, "app_reported_mi": 27.1, "ascent_ft_app": 1185.0, "ascent_ft_independent": 1180.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 280.2, "duplicate_points": 2, "repeat_frac": 0.075, "mirror_frac": 0.02, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "extend", "place": null, "places": null, "radius_m": 1000, "miles_delta": 6, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 extend 6mi: now 27.1 mi (+5.8 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'extend' vs allowed ['extend']", "value": null, "limit": null }, { "name": "parse.miles_delta", "status": "pass", "hard": true, "detail": "6 vs [5, 7]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1767 points, elevation present", "value": null, "limit": null }, { "name": "route.distance_delta", "status": "pass", "hard": true, "detail": "grew +5.79 mi, asked for +6.0 mi (tolerance +/-3 mi)", "value": 5.79, "limit": 6 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 280 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "8% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.075, "limit": 0.25 } ] }, { "id": "D29", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "can you run it past Olbrich Botanical Gardens", "start": "Olin Park, Madison WI", "why": "Via edit on an existing route -- 'past X' is a via, never an avoid.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 via Olbrich Botanical Gardens, Madison, WI: now 30.6 mi (+3.5 mi).", "notes": "", "seconds": 2.5, "measured": { "gpx_path": "evals/results/postfix/work/E1/route_22mi_ride_1_loop_brouter_edit2.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 2010, "distance_mi": 30.69, "app_reported_mi": 30.6, "ascent_ft_app": 1293.0, "ascent_ft_independent": 1290.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": { "Olbrich Botanical Gardens, Madison WI": 5.4 }, "avoid_on_road_m": {}, "max_point_gap_m": 280.2, "duplicate_points": 6, "repeat_frac": 0.121, "mirror_frac": 0.015, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "via", "place": null, "places": [ "Olbrich Botanical Gardens, Madison, WI" ], "radius_m": 1500, "miles_delta": null, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 via Olbrich Botanical Gardens, Madison, WI: now 30.6 mi (+3.5 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'via' vs allowed ['via']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 2010 points, elevation present", "value": null, "limit": null }, { "name": "route.via[Olbrich Botanical Gardens, Madison WI]", "status": "pass", "hard": true, "detail": "closest approach 5 m (limit 1500 m)", "value": 5.4, "limit": 1500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 280 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "12% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.121, "limit": 0.25 } ] }, { "id": "D30", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "that wasn't what I meant -- send it past the Memorial Union instead", "start": "Olin Park, Madison WI", "why": "Corrective phrasing must set revert_first, so the fix applies to the version BEFORE the bad change, not on top of it.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 via Memorial Union: now 27.1 mi (+0.0 mi).", "notes": "", "seconds": 2.0, "measured": { "gpx_path": "evals/results/postfix/work/E1/route_22mi_ride_1_loop_brouter_edit3.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1814, "distance_mi": 27.21, "app_reported_mi": 27.1, "ascent_ft_app": 1189.0, "ascent_ft_independent": 1199.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": { "Memorial Union, Madison WI": 15.3 }, "avoid_on_road_m": {}, "max_point_gap_m": 280.2, "duplicate_points": 4, "repeat_frac": 0.075, "mirror_frac": 0.02, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "via", "place": "Memorial Union", "places": null, "radius_m": 1500, "miles_delta": null, "target_miles": null, "connect_return": false, "revert_first": true }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 via Memorial Union: now 27.1 mi (+0.0 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'via' vs allowed ['via']", "value": null, "limit": null }, { "name": "parse.revert_first", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1814 points, elevation present", "value": null, "limit": null }, { "name": "route.via[Memorial Union, Madison WI]", "status": "pass", "hard": true, "detail": "closest approach 15 m (limit 1500 m)", "value": 15.3, "limit": 1500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 280 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "8% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.075, "limit": 0.25 } ] }, { "id": "D31", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "undo that", "start": "Olin Park, Madison WI", "why": "Plain undo, no replacement -- must parse as request_type 'undo', not as an edit.", "verdict": "pass", "fail_reasons": [], "summary": "Undone \u2014 back to route_22mi_ride_1_loop_brouter_edit1.gpx.", "notes": "", "seconds": 0.97, "measured": { "gpx_path": "evals/results/postfix/work/E1/route_22mi_ride_1_loop_brouter_edit1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1767, "distance_mi": 27.19, "app_reported_mi": 27.1, "ascent_ft_app": 1185.0, "ascent_ft_independent": 1180.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 280.2, "duplicate_points": 2, "repeat_frac": 0.075, "mirror_frac": 0.02, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": null, "parsed_interval": null, "request_type": "undo", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 1 candidate(s): Undone \u2014 back to route_22mi_ride_1_loop_brouter_edit1.gpx.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'undo', expected 'undo'", "value": "undo", "limit": "undo" }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1767 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 280 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "8% of the route rides the same road twice (the app rejects loops above 25%); mirror score 2% -- not an out-and-back", "value": 0.075, "limit": 0.25 } ] }, { "id": "D32", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "35 mile loop from Verona WI", "start": "Verona, WI", "why": "Session E2 setup.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 35.8 mi, 1844 ft, 0% repeat", "notes": "", "seconds": 9.02, "measured": { "gpx_path": "evals/results/postfix/work/E2/route_35mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 986, "distance_mi": 35.74, "app_reported_mi": 35.8, "ascent_ft_app": 1845.0, "ascent_ft_independent": 1948.0, "loop_closure_m": 0.0, "start_proximity_m": 1.9, "end_proximity_m": 1.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.7, "duplicate_points": 0, "repeat_frac": 0.002, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 35, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 35.8 mi, 1844 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "35 vs [32, 38]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 986 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "35.74 mi vs target 35 mi (+2.1%, tolerance +/-15%)", "value": 35.74, "limit": 35 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 2 m from the geocoded start (limit 600 m)", "value": 1.9, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 670 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.002, "limit": 0.25 } ] }, { "id": "D33", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "trim it down to 25 miles total", "start": "Verona, WI", "why": "An ABSOLUTE target, not a delta -- the service computes the delta from the route's current length.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 shorten 11mi: now 25.3 mi (-10.3 mi).", "notes": "", "seconds": 2.18, "measured": { "gpx_path": "evals/results/postfix/work/E2/route_35mi_ride_1_loop_brouter_edit1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 740, "distance_mi": 25.35, "app_reported_mi": 25.3, "ascent_ft_app": 1435.0, "ascent_ft_independent": 1478.0, "loop_closure_m": 0.0, "start_proximity_m": 1.9, "end_proximity_m": 1.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 616.8, "duplicate_points": 2, "repeat_frac": 0.003, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "shorten", "place": null, "places": null, "radius_m": 0, "miles_delta": null, "target_miles": 25, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 shorten 11mi: now 25.3 mi (-10.3 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.target_or_delta", "status": "pass", "hard": true, "detail": "target_miles=25 miles_delta=None", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 740 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "25.35 mi vs target 25 mi (+1.4%, tolerance +/-18%)", "value": 25.35, "limit": 25 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 617 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "D34", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "make this one start and finish at Mount Horeb WI", "start": "Verona, WI", "why": "Anchor: start AND end at one place as a single operation, not move_start alone.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 round trip from Mount Horeb, WI: now 44.1 mi (+18.8 mi).", "notes": "", "seconds": 2.02, "measured": { "gpx_path": "evals/results/postfix/work/E2/route_35mi_ride_1_loop_brouter_edit2.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1072, "distance_mi": 44.15, "app_reported_mi": 44.1, "ascent_ft_app": 2151.0, "ascent_ft_independent": 2195.0, "loop_closure_m": 0.0, "start_proximity_m": 3.5, "end_proximity_m": 3.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.7, "duplicate_points": 4, "repeat_frac": 0.207, "mirror_frac": 0.425, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "anchor", "place": "Mount Horeb, WI", "places": null, "radius_m": 1000, "miles_delta": null, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 round trip from Mount Horeb, WI: now 44.1 mi (+18.8 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'anchor' vs allowed ['anchor']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1072 points, elevation present", "value": null, "limit": null }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.anchored", "status": "pass", "hard": true, "detail": "start 4 m / finish 4 m from the anchor (limit 1500 m)", "value": 3.5, "limit": 1500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 670 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "21% of the route rides the same road twice (the app rejects loops above 25%); mirror score 42%", "value": 0.207, "limit": 0.25 } ] }, { "id": "D35", "split": "dev", "category": "edit", "tier": "live", "scope": "supported", "text": "actually keep me off County Road PB", "start": "Verona, WI", "why": "Avoid edit naming a county road -- exercises OSM way-geometry avoidance and the verified outcome banner.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 around County Road PB: now 47.7 mi (+3.7 mi). Verified clear of the area.", "notes": "", "seconds": 102.59, "measured": { "gpx_path": "evals/results/postfix/work/E2/route_35mi_ride_1_loop_brouter_edit3.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1134, "distance_mi": 47.8, "app_reported_mi": 47.7, "ascent_ft_app": 2423.0, "ascent_ft_independent": 2474.0, "loop_closure_m": 0.0, "start_proximity_m": 16780.3, "end_proximity_m": 16780.3, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.7, "duplicate_points": 6, "repeat_frac": 0.207, "mirror_frac": 0.395, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "avoid", "place": "County Road PB", "places": null, "radius_m": 1000, "miles_delta": null, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 2 candidate(s): Done \u2014 around County Road PB: now 47.7 mi (+3.7 mi). Verified clear of the area.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'avoid' vs allowed ['avoid']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1134 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 670 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "21% of the route rides the same road twice (the app rejects loops above 25%); mirror score 40%", "value": 0.207, "limit": 0.25 } ] }, { "id": "D36", "split": "dev", "category": "provider_failure", "tier": "offline", "scope": "supported", "text": "a 20 mile loop from Monona Terrace Madison WI", "start": "Monona Terrace, Madison WI", "why": "Routing provider unreachable. Must fail honestly and point at the routing server, not hand back an empty map or a traceback.", "verdict": "pass", "fail_reasons": [], "summary": "The routing server isn't responding, so no route could be generated. Nothing is wrong with your request \u2014 this is the router (BRouter) being down, and it will work again once the server is back.", "notes": "", "seconds": 2.72, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.declined", "status": "pass", "hard": true, "detail": "ok=False, 0 candidate(s): The routing server isn't responding, so no route could be generated. Nothing is wrong with your request \u2014 this is the router (BRouter) being down, and it will w", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: The routing server isn't responding, so no route could be generated. Nothing is wrong with your request \u2014 this is the router (BRouter) being down, and it will work again once the server is back.", "value": null, "limit": null }, { "name": "outcome.message_names_cause", "status": "pass", "hard": true, "detail": "user-visible summary names the cause (matched ['routing server', 'brouter'] of ['routing server', 'unreachable', 'brouter', 'router is', 'cannot reach']); the progress log matched ['routing server', 'unreachable', 'brouter']. Summary was: \"The routing server isn't responding, so no route could be generated. Nothing is wrong with your request \u2014 this is the router (BRouter) being down, and it will w\"", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" } ] }, { "id": "D37", "split": "dev", "category": "provider_failure", "tier": "offline", "scope": "supported", "text": "a 20 mile loop from Monona Terrace Madison WI", "start": "Monona Terrace, Madison WI", "why": "Geocoder unreachable. Probes whether a dead Nominatim surfaces as a usable message or as an unhandled exception.", "verdict": "pass", "fail_reasons": [], "summary": "Couldn't reach the address lookup service (OpenStreetMap Nominatim), so the request never got as far as routing. Nothing is wrong with what you asked for \u2014 try again in a moment. (GeocodeUnavailable)", "notes": "", "seconds": 2.05, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "avoid_places": [], "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.declined", "status": "pass", "hard": true, "detail": "ok=False, 0 candidate(s): Couldn't reach the address lookup service (OpenStreetMap Nominatim), so the request never got as far as routing. Nothing is wrong with what you asked for \u2014 try ", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: Couldn't reach the address lookup service (OpenStreetMap Nominatim), so the request never got as far as routing. Nothing is wrong with what you asked for \u2014 try again in a moment. (GeocodeUnavailable)", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" } ] }, { "id": "H01", "split": "heldout", "category": "simple_loop", "tier": "live", "scope": "supported", "text": "can I get a 15 mile loop out of Middleton WI", "start": "Middleton, WI", "why": "Held out: plain short loop, different town.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 15.2 mi, 678 ft, 0% repeat", "notes": "", "seconds": 8.0, "measured": { "gpx_path": "evals/results/postfix/work/H01/route_15mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 980, "distance_mi": 15.12, "app_reported_mi": 15.2, "ascent_ft_app": 678.0, "ascent_ft_independent": 647.0, "loop_closure_m": 0.0, "start_proximity_m": 22.9, "end_proximity_m": 22.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 49.5, "duplicate_points": 0, "repeat_frac": 0.003, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 15, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 15.2 mi, 678 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "15 vs [13, 17]", "value": null, "limit": null }, { "name": "parse.shape", "status": "pass", "hard": true, "detail": "'loop' vs allowed ['loop', 'both']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 980 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "15.12 mi vs target 15 mi (+0.8%, tolerance +/-15%)", "value": 15.12, "limit": 15 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 23 m from the geocoded start (limit 600 m)", "value": 22.9, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 50 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "H02", "split": "heldout", "category": "simple_loop", "tier": "live", "scope": "supported", "text": "looking for a ~18 mile ride from Stoughton WI, loop is fine", "start": "Stoughton, WI", "why": "Held out: tilde-approximate distance.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 17.9 mi, 625 ft, 0% repeat", "notes": "", "seconds": 3.37, "measured": { "gpx_path": "evals/results/postfix/work/H02/route_18mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 500, "distance_mi": 17.92, "app_reported_mi": 17.9, "ascent_ft_app": 619.0, "ascent_ft_independent": 619.0, "loop_closure_m": 0.0, "start_proximity_m": 1.1, "end_proximity_m": 1.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1077.8, "duplicate_points": 0, "repeat_frac": 0.0, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 18, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 17.9 mi, 625 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "18 vs [16, 20]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 500 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "17.92 mi vs target 18 mi (+0.4%, tolerance +/-15%)", "value": 17.92, "limit": 18 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 1 m from the geocoded start (limit 600 m)", "value": 1.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1078 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.0, "limit": 0.25 } ] }, { "id": "H03", "split": "heldout", "category": "distance_target", "tier": "live", "scope": "supported", "text": "two hours of riding from Sun Prairie WI", "start": "Sun Prairie, WI", "why": "Held out: duration converted to distance at the assumed 16 mph.", "verdict": "pass", "fail_reasons": [], "summary": "10 route(s) \u2014 best: #1 loop: 32.0 mi, 929 ft, 0% repeat", "notes": "Assumed 16 mph average pace for two-hour duration, yielding approximately 32 miles.", "seconds": 8.88, "measured": { "gpx_path": "evals/results/postfix/work/H03/route_32mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 728, "distance_mi": 32.0, "app_reported_mi": 32.0, "ascent_ft_app": 929.0, "ascent_ft_independent": 874.0, "loop_closure_m": 0.0, "start_proximity_m": 3.4, "end_proximity_m": 3.4, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1165.6, "duplicate_points": 0, "repeat_frac": 0.003, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 32, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 10 candidate(s): 10 route(s) \u2014 best: #1 loop: 32.0 mi, 929 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "32 vs [26, 38]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 728 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "32.00 mi vs target 32 mi (+0.0%, tolerance +/-15%)", "value": 32.0, "limit": 32 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 3 m from the geocoded start (limit 600 m)", "value": 3.4, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1166 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "H04", "split": "heldout", "category": "distance_target", "tier": "live", "scope": "supported", "text": "60 mile loop from Monona Terrace Madison WI", "start": "Monona Terrace, Madison WI", "why": "Held out: a long route -- generation cost and distance fit at the top of the normal range.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 62.5 mi, 3239 ft, 1% repeat", "notes": "", "seconds": 16.09, "measured": { "gpx_path": "evals/results/postfix/work/H04/route_60mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 2233, "distance_mi": 62.49, "app_reported_mi": 62.5, "ascent_ft_app": 3238.0, "ascent_ft_independent": 3265.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 592.2, "duplicate_points": 0, "repeat_frac": 0.007, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 60, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 62.5 mi, 3239 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "60 vs [55, 65]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 2233 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "62.49 mi vs target 60 mi (+4.2%, tolerance +/-15%)", "value": 62.49, "limit": 60 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 592 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.007, "limit": 0.25 } ] }, { "id": "H05", "split": "heldout", "category": "climb_max", "tier": "live", "scope": "supported", "text": "30 miles from Blue Mounds WI and make it hurt -- max elevation", "start": "Blue Mounds, WI", "why": "Held out: climb maximization in slang.", "verdict": "pass", "fail_reasons": [], "summary": "10 route(s) \u2014 best: #1 loop: 29.8 mi, 2244 ft, 0% repeat, 0.3 mi major", "notes": "", "seconds": 110.24, "measured": { "gpx_path": "evals/results/postfix/work/H05/route_30mi_maxclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1211, "distance_mi": 29.77, "app_reported_mi": 29.8, "ascent_ft_app": 2245.0, "ascent_ft_independent": 2341.0, "loop_closure_m": 0.0, "start_proximity_m": 6.2, "end_proximity_m": 6.2, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 669.0, "duplicate_points": 0, "repeat_frac": 0.002, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": true, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 10 candidate(s): 10 route(s) \u2014 best: #1 loop: 29.8 mi, 2244 ft, 0% repeat, 0.3 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [27, 33]", "value": null, "limit": null }, { "name": "parse.maximize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1211 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "29.77 mi vs target 30 mi (+0.8%, tolerance +/-15%)", "value": 29.77, "limit": 30 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 6 m from the geocoded start (limit 600 m)", "value": 6.2, "limit": 600.0 }, { "name": "route.climb_sought", "status": "pass", "hard": false, "detail": "2245 ft on a climb-maximizing request (expected at least 900 ft for this terrain)", "value": 2245.0, "limit": 900 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 669 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.002, "limit": 0.25 } ] }, { "id": "H06", "split": "heldout", "category": "climb_min", "tier": "live", "scope": "supported", "text": "flattest 35 miles you can find around Stoughton WI", "start": "Stoughton, WI", "why": "Held out: climb minimization as a superlative.", "verdict": "pass", "fail_reasons": [], "summary": "11 route(s) \u2014 best: #1 loop: 33.7 mi, 929 ft, 1% repeat, 0.1 mi major", "notes": "", "seconds": 5.72, "measured": { "gpx_path": "evals/results/postfix/work/H06/route_35mi_minclimb_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 537, "distance_mi": 33.71, "app_reported_mi": 33.7, "ascent_ft_app": 923.0, "ascent_ft_independent": 918.0, "loop_closure_m": 0.0, "start_proximity_m": 1.1, "end_proximity_m": 1.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1077.8, "duplicate_points": 0, "repeat_frac": 0.007, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 35, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": true, "shape": "both", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 11 candidate(s): 11 route(s) \u2014 best: #1 loop: 33.7 mi, 929 ft, 1% repeat, 0.1 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "35 vs [32, 38]", "value": null, "limit": null }, { "name": "parse.minimize_climb", "status": "pass", "hard": true, "detail": "True vs expected True", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 537 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "33.71 mi vs target 35 mi (+3.7%, tolerance +/-15%)", "value": 33.71, "limit": 35 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 1 m from the geocoded start (limit 600 m)", "value": 1.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1078 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.007, "limit": 0.25 } ] }, { "id": "H07", "split": "heldout", "category": "climb_min", "tier": "live", "scope": "supported", "text": "25 mile loop from Middleton WI, cap the climbing at 800 feet", "start": "Middleton, WI", "why": "Held out: a tight hard ceiling that may legitimately reject every candidate -- an honest refusal is a pass.", "verdict": "pass", "fail_reasons": [], "summary": "No route met the constraints \u2014 try a looser target or different distance.", "notes": "", "seconds": 7.59, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 25, "max_climb_ft": 800, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=False, 0 candidate(s): No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.max_climb_ft", "status": "pass", "hard": true, "detail": "800 vs [700, 900]", "value": null, "limit": null } ] }, { "id": "H08", "split": "heldout", "category": "via", "tier": "live", "scope": "supported", "text": "28 mile ride from Verona WI that swings through Paoli WI", "start": "Verona, WI", "why": "Held out: single waypoint, 'swings through' phrasing.", "verdict": "pass", "fail_reasons": [], "summary": "2 route(s) \u2014 best: #1 loop: 28.9 mi, 1642 ft, 1% repeat, 0.2 mi major", "notes": "", "seconds": 10.55, "measured": { "gpx_path": "evals/results/postfix/work/H08/route_28mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 862, "distance_mi": 28.85, "app_reported_mi": 28.9, "ascent_ft_app": 1641.0, "ascent_ft_independent": 1688.0, "loop_closure_m": 0.0, "start_proximity_m": 1.9, "end_proximity_m": 1.9, "via_distances_m": { "Paoli, WI": 47.3 }, "avoid_on_road_m": {}, "max_point_gap_m": 616.8, "duplicate_points": 0, "repeat_frac": 0.007, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 28, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [], "via_places": [ "Paoli, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): 2 route(s) \u2014 best: #1 loop: 28.9 mi, 1642 ft, 1% repeat, 0.2 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "28 vs [25, 31]", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 862 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "28.85 mi vs target 28 mi (+3.0%, tolerance +/-15%)", "value": 28.85, "limit": 28 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 2 m from the geocoded start (limit 600 m)", "value": 1.9, "limit": 600.0 }, { "name": "route.via[Paoli, WI]", "status": "pass", "hard": true, "detail": "closest approach 47 m (limit 2500 m)", "value": 47.3, "limit": 2500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 617 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.007, "limit": 0.25 } ] }, { "id": "H09", "split": "heldout", "category": "avoid", "tier": "live", "scope": "supported", "text": "25 mile loop from the Wisconsin State Capitol Madison WI, nothing on East Washington Avenue", "start": "Wisconsin State Capitol, Madison WI", "why": "Held out: avoid a major arterial that starts right at the start point.", "verdict": "pass", "fail_reasons": [], "summary": "5 route(s) \u2014 best: #1 loop: 24.8 mi, 971 ft, 0% repeat", "notes": "", "seconds": 9.54, "measured": { "gpx_path": "evals/results/postfix/work/H09/route_25mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1176, "distance_mi": 24.76, "app_reported_mi": 24.8, "ascent_ft_app": 971.0, "ascent_ft_independent": 942.0, "loop_closure_m": 0.0, "start_proximity_m": 0.1, "end_proximity_m": 0.1, "via_distances_m": {}, "avoid_on_road_m": { "East Washington Avenue, Madison WI": 14.3 }, "max_point_gap_m": 547.7, "duplicate_points": 0, "repeat_frac": 0.003, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [ "East Washington Avenue Madison WI" ], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 5 candidate(s): 5 route(s) \u2014 best: #1 loop: 24.8 mi, 971 ft, 0% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "25 vs [23, 27]", "value": null, "limit": null }, { "name": "parse.avoid_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1176 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "24.76 mi vs target 25 mi (+1.0%, tolerance +/-15%)", "value": 24.76, "limit": 25 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.1, "limit": 600.0 }, { "name": "route.avoid[East Washington Avenue, Madison WI]", "status": "pass", "hard": false, "detail": "14 m still ridden on it (limit 200 m)", "value": 14.3, "limit": 200.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 548 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "H10", "split": "heldout", "category": "multi_constraint", "tier": "live", "scope": "supported", "text": "I want 45 miles from Cross Plains WI, through Black Earth, under 2000 feet of climbing, and off Highway 14", "start": "Cross Plains, WI", "why": "Held out: the hardest combination in the set -- distance + waypoint + hard climb cap + avoid.", "verdict": "pass", "fail_reasons": [], "summary": "1 route(s) \u2014 best: #1 loop: 41.0 mi, 1991 ft, 0% repeat, 0.1 mi major", "notes": "", "seconds": 7.14, "measured": { "gpx_path": "evals/results/postfix/work/H10/route_45mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 2636, "distance_mi": 40.8, "app_reported_mi": 41.0, "ascent_ft_app": 1990.0, "ascent_ft_independent": 1906.0, "loop_closure_m": 0.0, "start_proximity_m": 17.2, "end_proximity_m": 17.2, "via_distances_m": { "Black Earth, WI": 475.2 }, "avoid_on_road_m": {}, "max_point_gap_m": 25.1, "duplicate_points": 1, "repeat_frac": 0.001, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 45, "max_climb_ft": 2000, "maximize_climb": false, "minimize_climb": false, "shape": "both", "avoid_places": [ "Highway 14, Wisconsin" ], "via_places": [ "Black Earth, Wisconsin" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 1 candidate(s): 1 route(s) \u2014 best: #1 loop: 41.0 mi, 1991 ft, 0% repeat, 0.1 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "45 vs [42, 48]", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "parse.max_climb_ft", "status": "pass", "hard": true, "detail": "2000 vs [1800, 2200]", "value": null, "limit": null }, { "name": "parse.avoid_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 2636 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "40.80 mi vs target 45 mi (+9.3%, tolerance +/-15%)", "value": 40.8, "limit": 45 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 17 m from the geocoded start (limit 600 m)", "value": 17.2, "limit": 600.0 }, { "name": "route.climb_cap", "status": "pass", "hard": true, "detail": "1990 ft vs cap 2000 ft (app model; independent unsmoothed sum 1906 ft)", "value": 1990.0, "limit": 2000 }, { "name": "route.via[Black Earth, WI]", "status": "pass", "hard": true, "detail": "closest approach 475 m (limit 2500 m)", "value": 475.2, "limit": 2500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 25 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.001, "limit": 0.25 } ] }, { "id": "H11", "split": "heldout", "category": "multi_constraint", "tier": "live", "scope": "supported", "text": "32 mile flat-ish loop from Sun Prairie WI that goes by Lake Mills WI", "start": "Sun Prairie, WI", "why": "Held out: soft flat preference plus a waypoint roughly a third of the ride away.", "verdict": "pass", "fail_reasons": [], "summary": "No route met the constraints \u2014 try a looser target or different distance.", "notes": "", "seconds": 3.66, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 32, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "avoid_places": [], "via_places": [ "Lake Mills, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=False, 0 candidate(s): No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: No route met the constraints \u2014 try a looser target or different distance.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "32 vs [29, 35]", "value": null, "limit": null }, { "name": "parse.via_places", "status": "pass", "hard": true, "detail": "1 item(s), need >= 1", "value": null, "limit": null } ] }, { "id": "H12", "split": "heldout", "category": "interval_spot", "tier": "live", "scope": "supported", "text": "somewhere within half an hour of Middleton WI to do 3x12 sweet spot", "start": "Middleton, WI", "why": "Held out: sweet-spot maps to a flat stretch; travel budget stated in words.", "verdict": "pass", "fail_reasons": [], "summary": "Found 3 interval spot(s) \u2014 best: #1: 4.0 mi @ -0.0%, 0 stops, 4.8 mi out. A 12-minute flat rep needs about 4.0 mi of road.", "notes": "User specified Middleton WI as the location for the interval spot search.", "seconds": 103.45, "measured": { "gpx_path": "evals/results/postfix/work/H12/spot_flat_3x12_1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 42, "distance_mi": 3.88, "app_reported_mi": 4.0, "ascent_ft_app": 77.0, "ascent_ft_independent": 83.0, "loop_closure_m": 5328.8, "start_proximity_m": 5709.4, "end_proximity_m": 10551.4, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 345.8, "duplicate_points": 0, "repeat_frac": 0.0, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": null, "parsed_interval": { "kind": "flat", "max_travel_minutes": 30, "rep_minutes": 12, "reps": 3 }, "request_type": "interval_spot", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 3 candidate(s): Found 3 interval spot(s) \u2014 best: #1: 4.0 mi @ -0.0%, 0 stops, 4.8 mi out. A 12-minute flat rep needs about 4.0 mi of road.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'interval_spot', expected 'interval_spot'", "value": "interval_spot", "limit": "interval_spot" }, { "name": "parse.reps", "status": "pass", "hard": true, "detail": "3 vs expected 3", "value": null, "limit": null }, { "name": "parse.rep_minutes", "status": "pass", "hard": true, "detail": "12 vs [10, 14]", "value": null, "limit": null }, { "name": "parse.kind", "status": "pass", "hard": true, "detail": "'flat' vs allowed ['flat']", "value": null, "limit": null }, { "name": "parse.max_travel_minutes", "status": "pass", "hard": true, "detail": "30 vs [25, 35]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 42 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 346 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.0, "limit": 0.25 }, { "name": "spot.within_travel_budget", "status": "pass", "hard": true, "detail": "stretch begins 5.7 km from the start; the stated budget of 30 min allows 12.1 km at the app's assumed 15 mph travel pace", "value": 5709.0, "limit": 12070.0 }, { "name": "spot.long_enough_for_a_rep", "status": "pass", "hard": false, "detail": "3.88 mi of road for a 12-minute flat rep, which needs about 4.00 mi at the app's assumed pace (1 lap(s) per rep)", "value": 3.88, "limit": 4.0 } ] }, { "id": "H13", "split": "heldout", "category": "ambiguous", "tier": "live", "scope": "supported", "text": "ride from Olin Park Madison WI, I've got maybe an hour but I might have longer", "start": "Olin Park, Madison WI", "why": "Held out: underspecified duration with a hedge. Probes whether the hedge is surfaced or silently resolved.", "verdict": "pass", "fail_reasons": [], "summary": "3 route(s) \u2014 best: #1 loop: 10.1 mi, 334 ft, 7% repeat", "notes": "Rider has about 1 hour available, possibly longer. Estimated 10 miles for 1-hour ride at 16 mph average; rider may request extension if more time available.", "seconds": 9.38, "measured": { "gpx_path": "evals/results/postfix/work/H13/route_10mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 666, "distance_mi": 10.12, "app_reported_mi": 10.1, "ascent_ft_app": 334.0, "ascent_ft_independent": 354.0, "loop_closure_m": 1.3, "start_proximity_m": 0.3, "end_proximity_m": 1.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 208.4, "duplicate_points": 0, "repeat_frac": 0.069, "mirror_frac": 0.14, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 10, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 3 candidate(s): 3 route(s) \u2014 best: #1 loop: 10.1 mi, 334 ft, 7% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "10 vs [10, 40]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 666 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "10.12 mi vs target 10 mi (+1.2%, tolerance +/-15%)", "value": 10.12, "limit": 10 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 208 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "7% of the route rides the same road twice (the app rejects loops above 25%); mirror score 14% -- not an out-and-back", "value": 0.069, "limit": 0.25 } ] }, { "id": "H14", "split": "heldout", "category": "ambiguous", "tier": "live", "scope": "supported", "text": "30 mile loop from Verona WI that goes through the place with the good pie", "start": "Verona, WI", "why": "Held out: an ungeocodable waypoint. A confidently wrong location is worse than a clear 'not found'.", "verdict": "pass", "fail_reasons": [], "summary": "Couldn't find the waypoint 'pie, Verona, WI' on the map. Try a road name plus the city, or a landmark that appears in OpenStreetMap.", "notes": "User mentioned 'the place with the good pie' - geocoding may need clarification on the specific bakery or restaurant name in Verona, WI", "seconds": 2.73, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": { "distance_miles": 30, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [ "pie, Verona, WI" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=False, 0 candidate(s): Couldn't find the waypoint 'pie, Verona, WI' on the map. Try a road name plus the city, or a landmark that appears in OpenStreetMap.", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: Couldn't find the waypoint 'pie, Verona, WI' on the map. Try a road name plus the city, or a landmark that appears in OpenStreetMap.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "30 vs [27, 33]", "value": null, "limit": null } ] }, { "id": "H15", "split": "heldout", "category": "infeasible", "tier": "live", "scope": "supported", "text": "a 300 mile loop from Monona Terrace Madison WI", "start": "Monona Terrace, Madison WI", "why": "Held out: beyond the server-side clamp of 150 mi. Probes whether the clamp is applied AND disclosed to the user.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 150.5 mi, 4398 ft, 0% repeat, 0.4 mi major", "notes": "", "seconds": 19.99, "measured": { "gpx_path": "evals/results/postfix/work/H15/route_150mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 3084, "distance_mi": 150.39, "app_reported_mi": 150.5, "ascent_ft_app": 4404.0, "ascent_ft_independent": 4379.0, "loop_closure_m": 0.0, "start_proximity_m": 14.6, "end_proximity_m": 14.6, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1394.2, "duplicate_points": 0, "repeat_frac": 0.005, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 300, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 150.5 mi, 4398 ft, 0% repeat, 0.4 mi major", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "300 vs [250, 350]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 3084 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "150.39 mi vs target 150 mi (+0.3%, tolerance +/-15%)", "value": 150.39, "limit": 150 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 14.6, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1394 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.005, "limit": 0.25 } ] }, { "id": "H16", "split": "heldout", "category": "out_of_scope", "tier": "live", "scope": "out_of_scope", "text": "give me the most scenic 40 mile loop from Mount Horeb WI, best views", "start": "Mount Horeb, WI", "why": "Held out: there is no scenery model. Honest behavior is a route plus a note that scenery was not used as a criterion.", "verdict": "pass", "fail_reasons": [], "summary": "1 route(s) \u2014 best: #1 loop: 38.5 mi, 2283 ft, 0% repeat", "notes": "User requests scenic route with best views; prioritize aesthetically beautiful roads and vistas", "seconds": 6.46, "measured": { "gpx_path": "evals/results/postfix/work/H16/route_40mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1075, "distance_mi": 38.53, "app_reported_mi": 38.5, "ascent_ft_app": 2284.0, "ascent_ft_independent": 2364.0, "loop_closure_m": 0.0, "start_proximity_m": 3.5, "end_proximity_m": 3.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 1196.2, "duplicate_points": 0, "repeat_frac": 0.004, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 40, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 1 candidate(s): 1 route(s) \u2014 best: #1 loop: 38.5 mi, 2283 ft, 0% repeat", "value": null, "limit": null }, { "name": "outcome.flagged_unsupported", "status": "pass", "hard": true, "detail": "unsupported part reported to the user: User requests scenic route with best views; prioritize aesthetically beautiful roads and vistas", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.notes", "status": "pass", "hard": true, "detail": "present", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1075 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "38.53 mi vs target 40 mi (+3.7%, tolerance +/-15%)", "value": 38.53, "limit": 40 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 4 m from the geocoded start (limit 600 m)", "value": 3.5, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 1196 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.004, "limit": 0.25 } ] }, { "id": "H17", "split": "heldout", "category": "out_of_scope", "tier": "live", "scope": "out_of_scope", "text": "25 mile loop from Middleton WI that avoids all stop signs and traffic lights", "start": "Middleton, WI", "why": "Held out: traffic-control avoidance exists for INTERVAL SPOTS only, never for routes. Easy to silently claim.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 25.3 mi, 1006 ft, 0% repeat", "notes": "User requests avoidance of stop signs and traffic lights; the tool prioritizes steady, low-interruption roads by default, but this preference is noted for emphasis.", "seconds": 8.15, "measured": { "gpx_path": "evals/results/postfix/work/H17/route_25mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1044, "distance_mi": 25.26, "app_reported_mi": 25.3, "ascent_ft_app": 1007.0, "ascent_ft_independent": 983.0, "loop_closure_m": 0.0, "start_proximity_m": 22.9, "end_proximity_m": 22.9, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 488.9, "duplicate_points": 0, "repeat_frac": 0.003, "mirror_frac": 0.005, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 25, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 25.3 mi, 1006 ft, 0% repeat", "value": null, "limit": null }, { "name": "outcome.flagged_unsupported", "status": "pass", "hard": true, "detail": "unsupported part reported to the user: User requests avoidance of stop signs and traffic lights; the tool prioritizes steady, low-interruption roads by default, but this preference is noted for emphasis.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.notes", "status": "pass", "hard": true, "detail": "present", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1044 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "25.26 mi vs target 25 mi (+1.0%, tolerance +/-15%)", "value": 25.26, "limit": 25 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 23 m from the geocoded start (limit 600 m)", "value": 22.9, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 489 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "0% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.003, "limit": 0.25 } ] }, { "id": "H18", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "20 mile loop from the Wisconsin State Capitol Madison WI", "start": "Wisconsin State Capitol, Madison WI", "why": "Held-out session E3 setup.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 20.1 mi, 862 ft, 1% repeat", "notes": "", "seconds": 12.82, "measured": { "gpx_path": "evals/results/postfix/work/E3/route_20mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1180, "distance_mi": 20.08, "app_reported_mi": 20.1, "ascent_ft_app": 861.0, "ascent_ft_independent": 917.0, "loop_closure_m": 0.0, "start_proximity_m": 0.1, "end_proximity_m": 0.1, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 382.6, "duplicate_points": 0, "repeat_frac": 0.009, "mirror_frac": 0.01, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 20.1 mi, 862 ft, 1% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "20 vs [18, 22]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1180 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "20.08 mi vs target 20 mi (+0.4%, tolerance +/-15%)", "value": 20.08, "limit": 20 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 0 m from the geocoded start (limit 600 m)", "value": 0.1, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 383 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "1% of the route rides the same road twice (the app rejects loops above 25%); mirror score 1% -- not an out-and-back", "value": 0.009, "limit": 0.25 } ] }, { "id": "H19", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "end the ride at Olbrich Botanical Gardens instead", "start": "Wisconsin State Capitol, Madison WI", "why": "Held out: move_end on a loop -- the finish moves, the start does not.", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 end at Olbrich Botanical Gardens, Madison, Wisconsin: now 23.5 mi (+3.4 mi).", "notes": "", "seconds": 2.12, "measured": { "gpx_path": "evals/results/postfix/work/E3/route_20mi_ride_1_loop_brouter_edit1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1411, "distance_mi": 23.51, "app_reported_mi": 23.5, "ascent_ft_app": 945.0, "ascent_ft_independent": 1012.0, "loop_closure_m": 4497.6, "start_proximity_m": 0.1, "end_proximity_m": 5.4, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 382.6, "duplicate_points": 2, "repeat_frac": 0.015, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "move_end", "place": "Olbrich Botanical Gardens, Madison, Wisconsin", "places": null, "radius_m": 1000, "miles_delta": null, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 end at Olbrich Botanical Gardens, Madison, Wisconsin: now 23.5 mi (+3.4 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'move_end' vs allowed ['move_end']", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1411 points, elevation present", "value": null, "limit": null }, { "name": "route.ends_where_asked", "status": "pass", "hard": true, "detail": "finishes 5 m from the requested end (limit 1200 m)", "value": 5.4, "limit": 1200 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 383 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.015, "limit": 0.25 } ] }, { "id": "H20", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "add about 8 more miles to it", "start": "Wisconsin State Capitol, Madison WI", "why": "Held out: extend applied to an already-edited route (chained edits must compose).", "verdict": "pass", "fail_reasons": [], "summary": "Done \u2014 extend 8mi: now 31.3 mi (+7.8 mi).", "notes": "", "seconds": 4.62, "measured": { "gpx_path": "evals/results/postfix/work/E3/route_20mi_ride_1_loop_brouter_edit2.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1747, "distance_mi": 31.33, "app_reported_mi": 31.3, "ascent_ft_app": 1196.0, "ascent_ft_independent": 1264.0, "loop_closure_m": 4497.6, "start_proximity_m": 0.1, "end_proximity_m": 4497.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 382.6, "duplicate_points": 4, "repeat_frac": 0.105, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": { "mode": "extend", "place": null, "places": null, "radius_m": 0, "miles_delta": 8, "target_miles": null, "connect_return": false, "revert_first": false }, "parsed_interval": null, "request_type": "edit_route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 2 candidate(s): Done \u2014 extend 8mi: now 31.3 mi (+7.8 mi).", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'edit_route', expected 'edit_route'", "value": "edit_route", "limit": "edit_route" }, { "name": "parse.mode", "status": "pass", "hard": true, "detail": "'extend' vs allowed ['extend']", "value": null, "limit": null }, { "name": "parse.miles_delta", "status": "pass", "hard": true, "detail": "8 vs [7, 9]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1747 points, elevation present", "value": null, "limit": null }, { "name": "route.distance_delta", "status": "pass", "hard": true, "detail": "grew +7.82 mi, asked for +8.0 mi (tolerance +/-4 mi)", "value": 7.82, "limit": 8 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 383 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "10% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.105, "limit": 0.25 } ] }, { "id": "H21", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "go back to the previous version", "start": "Wisconsin State Capitol, Madison WI", "why": "Held out: paraphrased undo.", "verdict": "pass", "fail_reasons": [], "summary": "Undone \u2014 back to route_20mi_ride_1_loop_brouter_edit1.gpx.", "notes": "", "seconds": 0.96, "measured": { "gpx_path": "evals/results/postfix/work/E3/route_20mi_ride_1_loop_brouter_edit1.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1411, "distance_mi": 23.51, "app_reported_mi": 23.5, "ascent_ft_app": 945.0, "ascent_ft_independent": 1012.0, "loop_closure_m": 4497.6, "start_proximity_m": 0.1, "end_proximity_m": 4497.5, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 382.6, "duplicate_points": 2, "repeat_frac": 0.015, "mirror_frac": 0.0, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": null, "parsed_edit": null, "parsed_interval": null, "request_type": "undo", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 1 candidate(s): Undone \u2014 back to route_20mi_ride_1_loop_brouter_edit1.gpx.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'undo', expected 'undo'", "value": "undo", "limit": "undo" }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1411 points, elevation present", "value": null, "limit": null }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 383 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 0% -- not an out-and-back", "value": 0.015, "limit": 0.25 } ] }, { "id": "H22", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "18 mile loop from the Memorial Union Madison WI", "start": "Memorial Union, Madison WI", "why": "Held-out session E4 setup.", "verdict": "pass", "fail_reasons": [], "summary": "4 route(s) \u2014 best: #1 loop: 18.5 mi, 698 ft, 6% repeat", "notes": "", "seconds": 12.22, "measured": { "gpx_path": "evals/results/postfix/work/E4/route_18mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 1194, "distance_mi": 18.37, "app_reported_mi": 18.5, "ascent_ft_app": 699.0, "ascent_ft_independent": 669.0, "loop_closure_m": 0.0, "start_proximity_m": 15.3, "end_proximity_m": 15.3, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": 37.9, "duplicate_points": 0, "repeat_frac": 0.058, "mirror_frac": 0.055, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 18, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "pass", "hard": true, "detail": "ok=True, 4 candidate(s): 4 route(s) \u2014 best: #1 loop: 18.5 mi, 698 ft, 6% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'route', expected 'route'", "value": "route", "limit": "route" }, { "name": "parse.distance_miles", "status": "pass", "hard": true, "detail": "18 vs [16, 20]", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 1194 points, elevation present", "value": null, "limit": null }, { "name": "route.distance", "status": "pass", "hard": true, "detail": "18.37 mi vs target 18 mi (+2.1%, tolerance +/-15%)", "value": 18.37, "limit": 18 }, { "name": "route.loop_closure", "status": "pass", "hard": true, "detail": "start-to-finish gap 0 m (limit 250 m)", "value": 0.0, "limit": 250.0 }, { "name": "route.starts_where_asked", "status": "pass", "hard": true, "detail": "begins 15 m from the geocoded start (limit 600 m)", "value": 15.3, "limit": 600.0 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 38 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "6% of the route rides the same road twice (the app rejects loops above 25%); mirror score 6% -- not an out-and-back", "value": 0.058, "limit": 0.25 } ] }, { "id": "H23", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "route me through Vilas Park and then the Arboretum", "start": "Memorial Union, Madison WI", "why": "Held out: a MULTI-waypoint via chain in one edit -- each place must arrive as its own geocodable string.", "verdict": "fail", "fail_reasons": [ "parse.request_type", "parse.mode", "parse.places_len_min" ], "summary": "6 route(s) \u2014 best: #1 loop: 13.9 mi, 560 ft, 2% repeat", "notes": "User did not specify total distance; defaulted to 15 miles for a loop visiting both locations in order.", "seconds": 17.53, "measured": { "gpx_path": "evals/results/postfix/work/E4/route_15mi_ride_1_loop_brouter.gpx", "gpx_valid": true, "gpx_error": null, "n_points": 781, "distance_mi": 13.9, "app_reported_mi": 13.9, "ascent_ft_app": 559.0, "ascent_ft_independent": 592.0, "loop_closure_m": 0.0, "start_proximity_m": 15.3, "end_proximity_m": 15.3, "via_distances_m": { "Vilas Park, Madison WI": 817.3, "UW Arboretum, Madison WI": 1017.9 }, "avoid_on_road_m": {}, "max_point_gap_m": 337.7, "duplicate_points": 0, "repeat_frac": 0.022, "mirror_frac": 0.03, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": true }, "parsed_route": { "distance_miles": 15, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "avoid_places": [], "via_places": [ "Vilas Park, Madison, Wisconsin", "University of Wisconsin Arboretum, Madison, Wisconsin" ] }, "parsed_edit": null, "parsed_interval": null, "request_type": "route", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.handled", "status": "pass", "hard": false, "detail": "ok=True, 6 candidate(s): 6 route(s) \u2014 best: #1 loop: 13.9 mi, 560 ft, 2% repeat", "value": null, "limit": null }, { "name": "parse.request_type", "status": "fail", "hard": true, "detail": "got 'route', expected 'edit_route'", "value": "route", "limit": "edit_route" }, { "name": "parse.mode", "status": "fail", "hard": true, "detail": "None vs allowed ['via']", "value": null, "limit": null }, { "name": "parse.places_len_min", "status": "fail", "hard": true, "detail": "0 place string(s): []", "value": null, "limit": null }, { "name": "gpx.valid", "status": "pass", "hard": true, "detail": "well-formed, 781 points, elevation present", "value": null, "limit": null }, { "name": "route.via[Vilas Park, Madison WI]", "status": "pass", "hard": true, "detail": "closest approach 817 m (limit 1500 m)", "value": 817.3, "limit": 1500 }, { "name": "route.via[UW Arboretum, Madison WI]", "status": "pass", "hard": true, "detail": "closest approach 1018 m (limit 1500 m)", "value": 1017.9, "limit": 1500 }, { "name": "geometry.continuous", "status": "pass", "hard": true, "detail": "continuous: no breaks. Longest single segment 338 m", "value": 0, "limit": 0 }, { "name": "geometry.repeat", "status": "pass", "hard": false, "detail": "2% of the route rides the same road twice (the app rejects loops above 25%); mirror score 3% -- not an out-and-back", "value": 0.022, "limit": 0.25 } ] }, { "id": "H24", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "undo", "start": "Memorial Union, Madison WI", "why": "Held out: bare one-word undo.", "verdict": "fail", "fail_reasons": [ "outcome.succeeded" ], "summary": "Nothing to undo \u2014 this is the earliest version.", "notes": "", "seconds": 0.97, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": null, "parsed_edit": null, "parsed_interval": null, "request_type": "undo", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.succeeded", "status": "fail", "hard": true, "detail": "ok=False, 0 candidate(s): Nothing to undo \u2014 this is the earliest version.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'undo', expected 'undo'", "value": "undo", "limit": "undo" } ] }, { "id": "H25", "split": "heldout", "category": "edit", "tier": "live", "scope": "supported", "text": "undo that", "start": "Monona Terrace, Madison WI", "why": "Held out: undo with NO route in the session. Must refuse cleanly rather than crash.", "verdict": "pass", "fail_reasons": [], "summary": "Nothing to undo \u2014 this is the earliest version.", "notes": "", "seconds": 0.0, "measured": { "gpx_path": null, "gpx_valid": null, "gpx_error": null, "n_points": null, "distance_mi": null, "app_reported_mi": null, "ascent_ft_app": null, "ascent_ft_independent": null, "loop_closure_m": null, "start_proximity_m": null, "end_proximity_m": null, "via_distances_m": {}, "avoid_on_road_m": {}, "max_point_gap_m": null, "duplicate_points": null, "repeat_frac": null, "mirror_frac": null, "teleports": [], "gaps_confirmed_road": [], "gaps_unknown": [], "has_elevation": null }, "parsed_route": null, "parsed_edit": null, "parsed_interval": null, "request_type": "undo", "error": null, "checks": [ { "name": "outcome.no_crash", "status": "pass", "hard": true, "detail": "handled without an unhandled exception", "value": null, "limit": null }, { "name": "outcome.declined", "status": "pass", "hard": true, "detail": "ok=False, 0 candidate(s): Nothing to undo \u2014 this is the earliest version.", "value": null, "limit": null }, { "name": "outcome.explained", "status": "pass", "hard": true, "detail": "user-facing explanation: Nothing to undo \u2014 this is the earliest version.", "value": null, "limit": null }, { "name": "parse.request_type", "status": "pass", "hard": true, "detail": "got 'undo', expected 'undo'", "value": "undo", "limit": "undo" } ] } ], "parse_variance": { "repeats": 5, "cases": 9, "cases_with_unstable_verdict": 2, "unstable": { "D20": [ "fail", "pass" ], "H23": [ "fail", "pass" ] }, "cases_with_spec_drift": 8, "cases_stable": 1, "field_drift": { "D01": [ { "address": "Monona Terrace, Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 12, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } }, { "address": "Monona Terrace, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 12, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } }, { "address": "Monona Terrace, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 12, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "via_places": [] } } ], "D10": [ { "address": "Monona Terrace Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 30, "max_climb_ft": 1000, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } }, { "address": "Monona Terrace, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 30, "max_climb_ft": 1000, "maximize_climb": false, "minimize_climb": true, "shape": "both", "via_places": [] } } ], "D15": [ { "address": "Memorial Union Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [ "Whitney Way Madison WI" ], "distance_miles": 22, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } }, { "address": "Memorial Union, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [ "Whitney Way, Madison, WI" ], "distance_miles": 22, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } } ], "D20": [ { "address": null, "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 0, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } }, { "address": null, "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 10, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } }, { "address": null, "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 15, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } }, { "address": null, "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 20, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } } ], "D21": [ { "address": "Olin Park Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 10, "max_climb_ft": 300, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "via_places": [] } }, { "address": "Olin Park, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 10, "max_climb_ft": 300, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "via_places": [] } }, { "address": "Olin Park, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 10, "max_climb_ft": 500, "maximize_climb": false, "minimize_climb": true, "shape": "loop", "via_places": [] } } ], "H10": [ { "address": "Cross Plains, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [ "Highway 14" ], "distance_miles": 45, "max_climb_ft": 2000, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [ "Black Earth, WI" ] } }, { "address": "Cross Plains, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [ "Highway 14, Wisconsin" ], "distance_miles": 45, "max_climb_ft": 2000, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [ "Black Earth, WI" ] } } ], "H13": [ { "address": "Olin Park Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 16, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "both", "via_places": [] } }, { "address": "Olin Park Madison WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 16, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } }, { "address": "Olin Park, Madison, WI", "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 10, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [] } } ], "H23": [ { "address": null, "edit": null, "interval": null, "request_type": "route", "route": { "avoid_places": [], "distance_miles": 15, "max_climb_ft": null, "maximize_climb": false, "minimize_climb": false, "shape": "loop", "via_places": [ "Vilas Park, Madison, Wisconsin", "University of Wisconsin\u2013Madison Arboretum, Madison, Wisconsin" ] } }, { "address": null, "edit": { "connect_return": false, "miles_delta": null, "mode": "via", "place": null, "places": [ "Vilas Park, Madison, Wisconsin", "Arboretum, Madison, Wisconsin" ], "radius_m": 1500, "revert_first": false, "target_miles": null }, "interval": null, "request_type": "edit_route", "route": null }, { "address": null, "edit": { "connect_return": false, "miles_delta": null, "mode": "via", "place": null, "places": [ "Vilas Park, Madison, Wisconsin", "University of Wisconsin Arboretum, Madison, Wisconsin" ], "radius_m": 1500, "revert_first": false, "target_miles": null }, "interval": null, "request_type": "edit_route", "route": null } ] }, "cases_with_semantic_drift": 7, "cases_hard_stable": 5, "per_case_hard_variants": { "D01": 1, "D10": 1, "D13": 1, "D15": 1, "D20": 4, "D21": 2, "H10": 1, "H13": 3, "H23": 2 }, "cases_semantically_stable": 2, "per_case_semantic_variants": { "D01": 2, "D10": 2, "D13": 1, "D15": 1, "D20": 4, "D21": 2, "H10": 2, "H13": 3, "H23": 3 }, "per_case_distinct_specs": { "D01": 3, "D10": 2, "D13": 1, "D15": 2, "D20": 4, "D21": 3, "H10": 2, "H13": 3, "H23": 3 }, "parse_cost_usd": 0.132955 }, "compare": { "against": "baseline", "against_overall": { "pass": 55, "partial": 0, "fail": 7, "inconclusive": 0, "total": 62 }, "against_by_split": { "dev": { "pass": 33, "partial": 0, "fail": 4, "inconclusive": 0, "total": 37 }, "heldout": { "pass": 22, "partial": 0, "fail": 3, "inconclusive": 0, "total": 25 } }, "against_live_by_split": { "dev": { "pass": 33, "partial": 0, "fail": 2, "inconclusive": 0, "total": 35 }, "heldout": { "pass": 22, "partial": 0, "fail": 3, "inconclusive": 0, "total": 25 } }, "changed": [ { "id": "D15", "category": "avoid", "split": "dev", "from": "fail", "to": "pass", "was": [ "outcome.succeeded", "parse.request_type", "parse.distance_miles", "parse.avoid_places" ], "now": [] }, { "id": "D18", "category": "interval_spot", "split": "dev", "from": "fail", "to": "partial", "was": [ "spot.lap_count_disclosed" ], "now": [ "spot.long_enough_for_a_rep" ] }, { "id": "D21", "category": "ambiguous", "split": "dev", "from": "pass", "to": "fail", "was": [], "now": [ "outcome.succeeded" ] }, { "id": "D36", "category": "provider_failure", "split": "dev", "from": "fail", "to": "pass", "was": [ "outcome.message_names_cause" ], "now": [] }, { "id": "D37", "category": "provider_failure", "split": "dev", "from": "fail", "to": "pass", "was": [ "outcome.no_crash" ], "now": [] }, { "id": "H14", "category": "ambiguous", "split": "heldout", "from": "fail", "to": "pass", "was": [ "outcome.no_crash" ], "now": [] } ], "improved": 5, "regressed": 1 } }