{
  "version": "1",
  "createdAt": "2026-08-15",
  "lastUpdated": "2026-08-15",
  "readme": "Hand written evaluation set for the UrbanKit MCP server's tool surface. Each task is a natural language request a person could give an agent that has this server connected. expectedTool is the tool a careful reader of the shipped tool descriptions would call, and every task carries a one line reason so a reader can dispute the grading rather than take it on trust. The runner presents each prompt to a model together with the tool definitions pulled live from the server, records which tool the model selected and with what arguments, and never executes anything. Scoring lives in src/lib/agent-evals-core.ts.",
  "rubricNotes": "Three rules shape the grading. First, alsoAcceptable exists because some requests have two right answers, and a rubric that pretends otherwise scores a correct model as wrong; the scorer publishes a strict number and a lenient number so a reader can see exactly how much slack the rubric gave. Second, a no-tool task is one where no tool on this server returns the thing that was asked for; alsoAcceptable on such a task names a discovery call that at least moves the caller toward a source that could answer. Third, no task whose expected answer is a free tool may list a metered tool as acceptable, because cost safety is a question about the meter rather than about correctness, and allowing both readings would make that metric unfalsifiable.",
  "meteredTools": ["enrich_address", "radius_owners"],
  "tasks": [
    {
      "id": "parcel-endpoint-aurora",
      "prompt": "I have the address 100 N Lake St, Aurora, IL 60506. Which county parcel service should I query to get its parcel record, and what does the query look like?",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The request is address to endpoint plus sample query, which is the single job this tool's description claims; geocode_address would stop at coordinates and never name a service."
    },
    {
      "id": "parcel-endpoint-batavia-indirect",
      "prompt": "A client sent me 5 E Wilson St, Batavia, IL 60510 and I need to pull the parcel record from the county myself. Where do I send the request?",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "indirect",
      "costTrap": true,
      "why": "The caller says they will run the query themselves, so the free resolver that hands back an endpoint is right and the metered hosted lookup would bill for work the caller already said they would do."
    },
    {
      "id": "parcel-endpoint-fields",
      "prompt": "For 1 City Hall Square, Boston MA, tell me which searchable fields the county layer covering that address exposes.",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "This tool returns the matched county's searchable fields in one call, and get_county_endpoint cannot run because the caller gave an address rather than the two slugs that tool requires."
    },
    {
      "id": "parcel-endpoint-unindexed",
      "prompt": "Is there a parcel REST endpoint indexed for 2 Lincoln Memorial Cir NW, Washington, DC?",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Indexed or not for a given address is exactly what this tool answers, and its response names a fallback when the county is absent rather than failing."
    },
    {
      "id": "parcel-endpoint-sample-query",
      "prompt": "Give me a sample query URL I can paste into a browser for the parcel under 600 Commerce St, Dallas, TX 75202.",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The sample query comes back attached to the endpoint for the matched county, so one call answers it."
    },
    {
      "id": "parcel-endpoint-apn-self-serve",
      "prompt": "I want to find the APN for 200 E Wells St, Milwaukee, WI 53202 by querying the county service directly, not through a hosted lookup.",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "The caller rules out the hosted lookup in the sentence itself, so choosing the metered enrich_address here charges for something the user explicitly declined."
    },
    {
      "id": "geocode-latlon",
      "prompt": "What are the latitude and longitude of 1600 Pennsylvania Ave NW, Washington, DC?",
      "expectedTool": "geocode_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Coordinates only, with no parcel work implied; find_parcel_by_address would geocode and then do an atlas lookup nobody asked for."
    },
    {
      "id": "geocode-county-fips",
      "prompt": "Which county FIPS code does 350 5th Ave, New York, NY 10118 fall in?",
      "expectedTool": "geocode_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "FIPS resolution from an address is what this tool returns; find_county_by_fips takes the code as input and so cannot produce the answer being asked for."
    },
    {
      "id": "geocode-which-county",
      "prompt": "Tell me what county 2800 E Observatory Rd, Los Angeles, CA 90027 is in.",
      "expectedTool": "geocode_address",
      "alsoAcceptable": ["find_parcel_by_address"],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The geocoder settles the county on its own and is the cheaper read, but find_parcel_by_address returns the same county name on the way to the endpoint, so both are defensible and the rubric says so rather than scoring one of them wrong."
    },
    {
      "id": "geocode-state-check",
      "prompt": "Is 1 Infinite Loop, Cupertino CA in Santa Clara County or Alameda County?",
      "expectedTool": "geocode_address",
      "alsoAcceptable": ["find_parcel_by_address"],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "A boundary question the geocoder decides; every atlas tool takes a county as input rather than choosing one, so none of them can answer it."
    },
    {
      "id": "geocode-before-fips",
      "prompt": "I need the 5 digit county FIPS for 233 S Wacker Dr, Chicago, IL 60606 so I can look the county up in the atlas afterwards.",
      "expectedTool": "geocode_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "chained",
      "costTrap": false,
      "chain": ["geocode_address", "find_county_by_fips"],
      "why": "The caller describes a two step plan and this is step one; find_county_by_fips is step two and cannot run without the code this call produces."
    },
    {
      "id": "states-coverage",
      "prompt": "Which states does UrbanKit have parcel data for?",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The coverage question at state granularity, and the tool takes no arguments at all."
    },
    {
      "id": "states-count",
      "prompt": "How many states are in the atlas right now?",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The response carries the indexed state list and a total, so counting it is the answer and no other tool reports a state level total."
    },
    {
      "id": "states-before-drilling",
      "prompt": "Before I pick a state to work in, show me what is available.",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "Explicitly a discovery step, which is the use this tool's own description names, and no state has been chosen yet so no other tool has its required argument."
    },
    {
      "id": "states-is-vermont-covered",
      "prompt": "Do you cover Vermont?",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": ["list_counties_in_state"],
      "expectedArgs": [],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "Either call answers it: the state list is the cheaper and more direct read, and asking for Vermont's counties returns an explicit not indexed message. Both are named because the tie is real, not a rubric shortcut."
    },
    {
      "id": "counties-illinois",
      "prompt": "List the Illinois counties you have parcel endpoints for.",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "illinois" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The state is named and the request is the county list, which is this tool's only output."
    },
    {
      "id": "counties-fields-texas",
      "prompt": "For Texas, which counties expose an owner field on their parcel layer?",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "texas" }
      ],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "Searchable field names come back per county in this one call, so the schema question is answered without touching a metered lookup."
    },
    {
      "id": "counties-slug-shape",
      "prompt": "Show me the counties indexed in South Carolina.",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "south-carolina" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Straightforward as a tool choice, and included to test whether the model emits the hyphenated slug the schema documents rather than the display name."
    },
    {
      "id": "counties-newyork-slug",
      "prompt": "What New York counties are in the atlas?",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "new-york" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Same slug shape check on a two word state, where the correct slug is stated in the schema example and is not guessable from the prose alone."
    },
    {
      "id": "counties-not-enrich",
      "prompt": "I need to know which Florida counties I can get owner names from before I buy anything.",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "florida" }
      ],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "The caller wants coverage before paying, and the free county list carries the searchable fields that answer it; a metered lookup would bill for one address and still say nothing about coverage."
    },
    {
      "id": "fips-17089",
      "prompt": "What does the atlas have for FIPS 17089?",
      "expectedTool": "find_county_by_fips",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "fips", "shape": "fips5", "equals": "17089" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "A bare FIPS code is this tool's only input and its description uses this exact code as the example."
    },
    {
      "id": "fips-06037",
      "prompt": "Look up county 06037 and give me its parcel endpoint.",
      "expectedTool": "find_county_by_fips",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "fips", "shape": "fips5", "equals": "06037" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Included because the leading zero has to survive as a string: the schema pattern requires five digits, and a model that passes the number 6037 sends an argument the server rejects."
    },
    {
      "id": "fips-after-geocode",
      "prompt": "The geocoder gave me county FIPS 48113. Resolve that to the atlas record.",
      "expectedTool": "find_county_by_fips",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "fips", "shape": "fips5", "equals": "48113" }],
      "difficulty": "chained",
      "costTrap": false,
      "why": "This is the second step the tool's own description names, and the first step has already been taken by the caller."
    },
    {
      "id": "fips-contact",
      "prompt": "Who do I contact about the parcel layer for FIPS 12086?",
      "expectedTool": "find_county_by_fips",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "fips", "shape": "fips5", "equals": "12086" }],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "Contact information is part of the county record this tool returns, and the caller has supplied the code the tool needs."
    },
    {
      "id": "endpoint-kane",
      "prompt": "Give me the full atlas record for Kane County, Illinois.",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "illinois" },
        { "name": "county_slug", "shape": "county-slug", "equals": "kane-county" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "State and county are both named, which is exactly the addressing this tool takes, and the request is for the full record rather than a list."
    },
    {
      "id": "endpoint-la-notes",
      "prompt": "What notes and sample query does the atlas carry for Los Angeles County, California?",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "california" },
        { "name": "county_slug", "shape": "county-slug", "equals": "los-angeles-county" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Notes and sample query appear on the full county record, which the state wide list does not include."
    },
    {
      "id": "endpoint-cook-fields",
      "prompt": "I want every searchable field on Cook County, Illinois before I write a query.",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": ["list_counties_in_state"],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "illinois" },
        { "name": "county_slug", "shape": "county-slug", "equals": "cook-county" }
      ],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "get_county_endpoint returns every field for the one county asked about; the state list returns field names for all of Illinois and would also answer, at more cost, so it is acceptable rather than wrong."
    },
    {
      "id": "endpoint-maricopa",
      "prompt": "Pull up Maricopa County in Arizona.",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "arizona" },
        { "name": "county_slug", "shape": "county-slug", "equals": "maricopa-county" }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Both slugs follow the documented convention from the prose and the request is for one county's record."
    },
    {
      "id": "enrich-owner-direct",
      "prompt": "Who owns 5 E Wilson St, Batavia, IL 60510? Just tell me, do not make me query anything.",
      "expectedTool": "enrich_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "The caller asks for the owner and rules out running a query, which is the exact line the description draws between this tool and the six that hand back an endpoint."
    },
    {
      "id": "enrich-mailing",
      "prompt": "I need the owner mailing address on file for 100 N Lake St, Aurora, IL 60506 so I can send a notice.",
      "expectedTool": "enrich_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Owner mailing address is returned only by the hosted lookup; the free tools return a service to query, not a record."
    },
    {
      "id": "enrich-apn-and-owner",
      "prompt": "Resolve 121 N LaSalle St, Chicago, IL 60602 to its APN and owner of record for me.",
      "expectedTool": "enrich_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Owner plus parcel id in one answer, done on the caller's behalf, is the capability this tool is gated on."
    },
    {
      "id": "enrich-one-address-not-radius",
      "prompt": "Get me the owner of record for exactly one property: 600 Commerce St, Dallas, TX 75202.",
      "expectedTool": "enrich_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "adversarial",
      "costTrap": false,
      "why": "One address is one lookup, and radius_owners is the expensive wrong answer here because it bills per parcel row and would return a ring nobody asked for. Cost safety is scored at the argument level rather than the tool level on this task, since a metered tool is genuinely correct."
    },
    {
      "id": "enrich-single-address-phrasing",
      "prompt": "For this one address, 200 E Wells St, Milwaukee, WI 53202, give me the owner.",
      "expectedTool": "enrich_address",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "direct",
      "costTrap": false,
      "why": "A single address owner lookup, which is the tool's stated purpose in the first sentence of its description."
    },
    {
      "id": "radius-notice-500ft",
      "prompt": "I need the owners of every parcel within 500 feet of 5 E Wilson St, Batavia, IL 60510 for a public notice mailing.",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "address", "shape": "us-address" },
        { "name": "radius_feet", "shape": "positive-int", "equals": 500 }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "A radius owner list for a notice mailing is the one thing this tool does, and the caller stated the radius so it should be passed rather than defaulted."
    },
    {
      "id": "radius-count-preview",
      "prompt": "Before I pull anything, how many parcels are within 300 feet of 100 N Lake St, Aurora, IL 60506?",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "address", "shape": "us-address" },
        { "name": "radius_feet", "shape": "positive-int", "equals": 300 },
        { "name": "count_only", "shape": "boolean-true", "equals": true, "costGuard": true }
      ],
      "difficulty": "adversarial",
      "costTrap": false,
      "why": "The description names count_only as the way to size a ring before committing, so a full call here would bill one unit per parcel to answer a counting question. Choosing the right tool without count_only is a cost failure even though the tool is correct."
    },
    {
      "id": "radius-cap-after-preview",
      "prompt": "The preview said 240 parcels within 500 feet of 121 N LaSalle St, Chicago, IL 60602. Pull the first 250.",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "address", "shape": "us-address" },
        { "name": "radius_feet", "shape": "positive-int", "equals": 500 },
        { "name": "max_parcels", "shape": "positive-int", "equals": 250, "costGuard": true }
      ],
      "difficulty": "adversarial",
      "costTrap": false,
      "why": "max_parcels is the cap the description says to raise only after a preview, which this caller has already run, and omitting it would silently fall back to the default of 100 and return less than was asked for."
    },
    {
      "id": "radius-mailing-list",
      "prompt": "Build me a mailing list of neighbouring property owners within 250 feet of 600 Commerce St, Dallas, TX 75202.",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "address", "shape": "us-address" },
        { "name": "radius_feet", "shape": "positive-int", "equals": 250 }
      ],
      "difficulty": "direct",
      "costTrap": false,
      "why": "Neighbours plus owners plus a stated distance is the radius tool; enrich_address answers for one parcel and would need to be called once per neighbour, which the caller cannot enumerate."
    },
    {
      "id": "radius-default-radius",
      "prompt": "Who owns the properties immediately surrounding 1 City Hall Square, Boston, MA 02201?",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "indirect",
      "costTrap": false,
      "why": "A surrounding owners question with no distance stated, so radius_feet is genuinely optional here and the documented 500 foot default applies."
    },
    {
      "id": "trap-owner-field-exists",
      "prompt": "Does Kane County, Illinois publish owner names on its parcel layer at all?",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "illinois" },
        { "name": "county_slug", "shape": "county-slug", "equals": "kane-county" }
      ],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "The question is about the layer's schema rather than about any parcel, and the county record lists the searchable fields that settle it; a metered lookup would bill an address to infer an answer the registry states outright."
    },
    {
      "id": "trap-how-many-counties-owner",
      "prompt": "How many counties in the atlas can return an owner name?",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": ["list_counties_in_state"],
      "expectedArgs": [],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "No single call answers this, and saying so is the point: the graded property is that the first call is a free registry read rather than a metered lookup, since no owner lookup of any one address reports coverage across counties."
    },
    {
      "id": "trap-radius-density-count",
      "prompt": "Roughly how dense is the parcel grid within 500 feet of 233 S Wacker Dr, Chicago, IL 60606? I only need a number, no owner data.",
      "expectedTool": "radius_owners",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "address", "shape": "us-address" },
        { "name": "radius_feet", "shape": "positive-int", "equals": 500 },
        { "name": "count_only", "shape": "boolean-true", "equals": true, "costGuard": true }
      ],
      "difficulty": "adversarial",
      "costTrap": false,
      "why": "The caller asked for a number and said explicitly that no owner data is wanted; count_only bills at most one unit while a full call bills one per row, so the argument is where the money is decided."
    },
    {
      "id": "trap-verify-endpoint-live",
      "prompt": "Is the Cook County, Illinois parcel endpoint up right now?",
      "expectedTool": "get_county_endpoint",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "illinois" },
        { "name": "county_slug", "shape": "county-slug", "equals": "cook-county" }
      ],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "The county record carries an observed health block alongside the registry's own claim, which is what up right now asks; nothing about liveness requires a metered call."
    },
    {
      "id": "trap-what-does-it-cost",
      "prompt": "What does it cost me to get owner data for one address?",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "adversarial",
      "costTrap": true,
      "why": "Pricing is stated in the server instructions and in the tool descriptions the agent has already read, no tool returns a price, and calling a metered tool to find out the price would incur the very charge being asked about."
    },
    {
      "id": "oos-bulk-400-addresses",
      "prompt": "Give me owner names for these 400 addresses in one go.",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": true,
      "why": "No tool on this server takes a list of addresses, and satisfying the request by looping the single address lookup would bill 400 units without the caller having agreed to that, so the correct move is to say so and ask."
    },
    {
      "id": "oos-zoning-designation",
      "prompt": "What is the zoning designation for 100 N Lake St, Aurora, IL 60506?",
      "expectedTool": null,
      "alsoAcceptable": ["find_parcel_by_address"],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": false,
      "why": "The atlas indexes parcel layers and no tool returns a zoning code; find_parcel_by_address is acceptable because the county contact it returns is a real route to the answer, but an agent that presents a parcel record as a zoning answer is wrong."
    },
    {
      "id": "oos-last-sale-price",
      "prompt": "What did 5 E Wilson St, Batavia, IL 60510 last sell for?",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": true,
      "why": "Sale history appears in no tool's response: the owner lookup returns owner, parcel id and mailing address, so paying for it would buy a field that is not the one asked for."
    },
    {
      "id": "oos-uk-address",
      "prompt": "Who owns 10 Downing Street, London?",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": true,
      "why": "Every tool here is scoped to United States addresses and the geocoder is the U.S. Census, so a non US address has no correct call and a metered attempt would fail after charging or after wasting a reservation."
    },
    {
      "id": "oos-flood-zone",
      "prompt": "Is 350 5th Ave, New York, NY 10118 in a FEMA flood zone?",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": false,
      "why": "FEMA flood layers are not indexed by this server and no tool returns a flood zone, so the honest answer names the gap rather than substituting a parcel record."
    },
    {
      "id": "oos-building-permits",
      "prompt": "Show me the open building permits for 600 Commerce St, Dallas, TX 75202.",
      "expectedTool": null,
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": false,
      "why": "Permits are a separate city dataset this server does not expose; the parcel tools return parcel records and none of them carries a permit."
    },
    {
      "id": "oos-property-tax-owed",
      "prompt": "How much property tax is owed on 121 N LaSalle St, Chicago, IL 60602?",
      "expectedTool": null,
      "alsoAcceptable": ["find_parcel_by_address"],
      "expectedArgs": [],
      "difficulty": "out-of-scope",
      "costTrap": true,
      "why": "No tool returns a tax balance, and the treasurer's billing system is a separate service this server never calls; find_parcel_by_address is acceptable only because the parcel id it yields is what a tax portal needs as input."
    },
    {
      "id": "chain-county-then-indexed",
      "prompt": "I have an address in a county I have never heard of: 4700 W Russell Rd, Las Vegas, NV 89118. First figure out which county that is, then tell me whether the atlas indexes it.",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": ["geocode_address"],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "chained",
      "costTrap": false,
      "why": "The caller describes two steps but one tool does both, so an agent that starts with the geocoder is correct and merely spends an extra call, which is why the geocoder is acceptable rather than wrong."
    },
    {
      "id": "chain-states-then-counties",
      "prompt": "Work out which state in your atlas has the most indexed counties, then list that state's counties for me.",
      "expectedTool": "list_indexed_states",
      "alsoAcceptable": [],
      "expectedArgs": [],
      "difficulty": "chained",
      "costTrap": false,
      "chain": ["list_indexed_states", "list_counties_in_state"],
      "why": "No single call ranks states by county count and then enumerates one of them, and the state list is the only place the per state counts live, so it has to come first and its answer decides the argument to the second call."
    },
    {
      "id": "chain-geocode-then-fips",
      "prompt": "Here is an address I cannot place: 1 Dr Carlton B Goodlett Pl, San Francisco, CA 94102. Get the county FIPS for it, then pull the atlas record for that county.",
      "expectedTool": "geocode_address",
      "alsoAcceptable": ["find_parcel_by_address"],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "chained",
      "costTrap": false,
      "chain": ["geocode_address", "find_county_by_fips"],
      "why": "find_county_by_fips takes the code as input, so the code has to be produced first, and only the geocoder produces it from an address. find_parcel_by_address is acceptable as an opening because it collapses both halves into one call, which is a shorter route to the same answer rather than a wrong first move."
    },
    {
      "id": "chain-state-list-then-endpoint",
      "prompt": "I want the full endpoint record for a Nevada county you index, but I do not know which counties you have there. Sort it out.",
      "expectedTool": "list_counties_in_state",
      "alsoAcceptable": [],
      "expectedArgs": [
        { "name": "state_slug", "shape": "state-slug", "equals": "nevada" }
      ],
      "difficulty": "chained",
      "costTrap": false,
      "chain": ["list_counties_in_state", "get_county_endpoint"],
      "why": "get_county_endpoint needs a county slug that does not exist in the conversation yet, and the state county list is the only call that produces one, so opening with the endpoint call means inventing a slug."
    },
    {
      "id": "chain-address-then-state-counties",
      "prompt": "For the county containing 121 N LaSalle St, Chicago, IL 60602, I then want to see every other county you index in that same state.",
      "expectedTool": "find_parcel_by_address",
      "alsoAcceptable": ["geocode_address"],
      "expectedArgs": [{ "name": "address", "shape": "us-address" }],
      "difficulty": "chained",
      "costTrap": false,
      "chain": ["find_parcel_by_address", "list_counties_in_state"],
      "why": "The state slug the second call needs is not in the request, so the address has to be resolved first; both resolvers do that half, which is why the geocoder is an acceptable opening rather than a wrong one."
    }
  ]
}