{
  "eval_id": "B5-2026-07-11",
  "generated_at": "2026-07-11",
  "source": "evidence/tool-evals/cases.json, evidence/tool-evals/results.md, evidence/tool-evals/transcripts.md (internal eval workspace; not itself published) -- every case id, prompt, arg, and score below is copied from those files.",
  "method": "Anthropic tool-eval rubric, three axes, graded by three fresh-context subagents. Picker sees ONLY the seven tool names + descriptions and a realistic user ask, and must name the right tool with no args. First-call-success sees names + descriptions + params and must emit the exact first call (tool + JSON args), validated against the frozen zod schemas. Response-usability sees a real production response envelope and must answer the user's question from it and name its next action. No agent saw the expected answers.",
  "models": {
    "picker": "Haiku (the floor host)",
    "first_call": "Sonnet",
    "response_usability": "Sonnet"
  },
  "limitations": "This eval measures tool-selection and first-call / response-envelope usability only: whether a model picks the right tool for a realistic ask, emits a valid first call, and can read and act on a real response envelope. It does NOT measure LinkedIn data quality, coverage, or freshness, and it does NOT measure API availability or latency.",
  "picker": {
    "description": "Model sees only the seven tool names + descriptions; must name the right tool for a realistic ask.",
    "score": "14/14",
    "cases": [
      {
        "id": "P1",
        "ask": "Find the LinkedIn profile of the CTO of Datadog.",
        "expected": [
          "search_people"
        ],
        "selected": "search_people",
        "pass": true
      },
      {
        "id": "P2",
        "ask": "What has Anthropic been posting on LinkedIn recently?",
        "expected": [
          "get_posts"
        ],
        "selected": "get_posts",
        "pass": true
      },
      {
        "id": "P3",
        "ask": "Pull the full work history for linkedin.com/in/satyanadella.",
        "expected": [
          "get_profile"
        ],
        "selected": "get_profile",
        "pass": true
      },
      {
        "id": "P4",
        "ask": "How many employees does Stripe have on LinkedIn?",
        "expected": [
          "get_company"
        ],
        "selected": "get_company",
        "pass": true
      },
      {
        "id": "P5",
        "ask": "A user pasted this link, what is it? https://www.linkedin.com/company/openai/?utm_source=chatgpt",
        "expected": [
          "resolve_url",
          "get_company"
        ],
        "selected": "resolve_url",
        "pass": true
      },
      {
        "id": "P6",
        "ask": "I don't have a Veezee API key yet. Get me set up.",
        "expected": [
          "provision"
        ],
        "selected": "provision",
        "pass": true
      },
      {
        "id": "P7",
        "ask": "How many credits do I have left?",
        "expected": [
          "get_usage"
        ],
        "selected": "get_usage",
        "pass": true
      },
      {
        "id": "P8",
        "ask": "Find software engineers at Shopify who used to work at Amazon.",
        "expected": [
          "search_people"
        ],
        "selected": "search_people",
        "pass": true
      },
      {
        "id": "P9",
        "ask": "Get the education history of williamhgates.",
        "expected": [
          "get_profile"
        ],
        "selected": "get_profile",
        "pass": true
      },
      {
        "id": "P10",
        "ask": "I only have the website vercel.com. Get their LinkedIn company info.",
        "expected": [
          "get_company"
        ],
        "selected": "get_company",
        "pass": true
      },
      {
        "id": "P11",
        "ask": "Did Bill Gates post anything about malaria this month?",
        "expected": [
          "get_posts"
        ],
        "selected": "get_posts",
        "pass": true
      },
      {
        "id": "P12",
        "ask": "Look up the profile of the person named 'Jordan Rivera' who works at Datadog.",
        "expected": [
          "search_people"
        ],
        "selected": "search_people",
        "pass": true
      },
      {
        "id": "P13",
        "ask": "Check whether my key is still valid and what plan I'm on.",
        "expected": [
          "get_usage"
        ],
        "selected": "get_usage",
        "pass": true
      },
      {
        "id": "P14",
        "ask": "Read this specific post for me: https://www.linkedin.com/feed/update/urn:li:activity:7123456789/",
        "expected": [
          "resolve_url",
          "none"
        ],
        "selected": "none",
        "pass": true
      }
    ]
  },
  "first_call_success": {
    "description": "Model sees names + descriptions + params; must emit the exact first call (tool + JSON args), validated against the frozen zod schemas.",
    "score": "7/7 valid (after any description iteration shown below), 6/7 optimal on the very first try",
    "cases": [
      {
        "id": "F1",
        "ask": "Pull the work history for linkedin.com/in/satyanadella.",
        "expected_tool": "get_profile",
        "tool": "get_profile",
        "args": {
          "identifier": "linkedin.com/in/satyanadella",
          "sections": [
            "experience"
          ]
        },
        "optimal_first_try": true,
        "pass": true
      },
      {
        "id": "F2",
        "ask": "Find software engineers at Shopify who used to work at Amazon.",
        "expected_tool": "search_people",
        "tool": "search_people",
        "args": {
          "title": "Software Engineer",
          "current_company": "Shopify",
          "past_company": "Amazon"
        },
        "optimal_first_try": true,
        "pass": true
      },
      {
        "id": "F3",
        "ask": "I only have the website vercel.com. Get their LinkedIn company info.",
        "expected_tool": "get_company",
        "tool": "get_company",
        "args": {
          "identifier": "vercel"
        },
        "optimal_first_try": true,
        "note": "Pre-WAVE3 catalog: resolved to the guessed slug 'vercel', not the raw domain. Superseded by the WAVE3 domain-emulation rerun below (identifier='vercel.com' directly, zero extra turns).",
        "pass": true
      },
      {
        "id": "F4",
        "ask": "What has Anthropic been posting on LinkedIn recently?",
        "expected_tool": "get_posts",
        "tool": "get_posts",
        "args": {
          "identifier": "anthropic"
        },
        "optimal_first_try": true,
        "pass": true
      },
      {
        "id": "F5",
        "ask": "Get the education history of williamhgates.",
        "expected_tool": "get_profile",
        "tool": "get_profile",
        "args": {
          "identifier": "williamhgates",
          "sections": [
            "education"
          ]
        },
        "optimal_first_try": true,
        "pass": true
      },
      {
        "id": "F6",
        "ask": "Search up to 30 results for CTOs at Anthropic, but do not spend more than 15 credits on the call.",
        "expected_tool": "search_people",
        "tool": "search_people",
        "optimal_first_try": false,
        "note": "The one systematic weakness this eval found. v1 and v2 tool descriptions let the model combine a company NAME filter with limit=30, which the planner rejects (name resolution plus 3 pages exceeds the internal fetch budget): recoverable via the server error, but a first-call failure. Fixed in v3 with an imperative description rule; the server error remains the safety net for agents that ignore it.",
        "iterations": [
          {
            "version": "v1 (no caveat)",
            "args": {
              "title": "CTO",
              "current_company": "Anthropic",
              "limit": 30,
              "max_credits": 15
            },
            "outcome": "rejected by the planner (name filter + limit=30 combination)"
          },
          {
            "version": "v2 (polite caveat added to description)",
            "args": {
              "title": "CTO",
              "current_company": "Anthropic",
              "limit": 30,
              "max_credits": 15
            },
            "outcome": "unchanged: same rejected combination"
          },
          {
            "version": "v3 (imperative caveat: Do NOT combine a company NAME filter with limit=30...)",
            "args": {
              "keywords": "CTO Anthropic",
              "title": "CTO",
              "limit": 30,
              "max_credits": 15
            },
            "outcome": "valid: avoided the rejected combination, quote 12 <= max_credits 15; shipped as the live description"
          }
        ],
        "pass": true
      },
      {
        "id": "F7-A5",
        "ask": "HISTORICAL (pre-WAVE3): get_company identifier='microsoft.com' returns a teaching INVALID_INPUT; the agent must produce the corrected call in one turn.",
        "expected_tool": "get_company",
        "tool": "get_company",
        "args": {
          "identifier": "microsoft"
        },
        "optimal_first_try": true,
        "status": "Historical; superseded 2026-07-11 by the WAVE3 F3 domain-emulation change, under which 'microsoft.com' now resolves directly with zero extra turns (see the rerun below).",
        "error_envelope_seen": {
          "status": 400,
          "code": "INVALID_INPUT",
          "message": "'microsoft.com' looks like a website domain, which is not a LinkedIn identifier. identifier accepts a LinkedIn company URL, the slug after /company/, or a numeric id. Did you mean identifier='microsoft'?"
        },
        "pass": true
      }
    ]
  },
  "response_usability": {
    "description": "Model sees a real production response envelope; must answer the user's question from it and name its next action.",
    "score": "3/3",
    "cases": [
      {
        "id": "U1",
        "ask": "How many employees does Microsoft have on LinkedIn, and what industry is it in?",
        "envelope_source": "production get_company identifier=microsoft",
        "answer": "Microsoft has about 233,334 employees on LinkedIn (listed in the 10001+ range), and its industry is Software Development.",
        "grading_note": "Both facts extracted correctly from the envelope's employee_count / employee_count_range / industry fields.",
        "pass": true
      },
      {
        "id": "U2",
        "ask": "How many credits do I have left and what did my last few calls cost?",
        "envelope_source": "production get_usage",
        "answer": "You have 486 credits remaining. Your last two calls: search_people cost 10 credits, and get_company cost 4 credits (both on 2026-07-10T19:57:07Z).",
        "grading_note": "Balance and per-call costs with receipt dates extracted correctly from balance_remaining and recent_receipts.",
        "pass": true
      },
      {
        "id": "U3",
        "ask": "Find CTOs at Anthropic; I will want their profiles next.",
        "envelope_source": "production search_people keywords=CTO current_company=1035 limit=10",
        "answer": "These results don't look like Anthropic CTOs. The search's current_company=1035 is Microsoft's company id (matching the get_company result for Microsoft), not Anthropic's, so the search appears to have run against the wrong company filter. Of the 5 results, three are Microsoft/Xbox-affiliated and two are CTOs of unrelated small startups; none show an Anthropic affiliation. Next action: re-run search_people with the correct Anthropic company id, not 1035, before spending further credits on profile fetches.",
        "grading_note": "The eval prompt itself mislabeled the search (the envelope was actually a Microsoft-id (1035) search mislabeled 'Anthropic'). The agent caught the mismatch from the envelope's own echoed current_company field, refused to spend credits fetching wrong-company profiles, and named the correct next action.",
        "pass": true
      }
    ]
  },
  "wave3_f3_domain_rerun": {
    "date": "2026-07-11",
    "context": "After the get_company description change that made website domains first-class identifiers (WAVE3 F3), the affected picker and first-call cases were rerun.",
    "picker_score": "3/3",
    "picker_cases": [
      {
        "ask": "bare domain (vercel.com)",
        "selected": "get_company",
        "pass": true
      },
      {
        "ask": "employee-count-behind-domain (stripe.com)",
        "selected": "get_company",
        "pass": true
      },
      {
        "ask": "subdomain paste (news.microsoft.com)",
        "selected": "get_company",
        "pass": true,
        "note": "the answer cited the verified-or-INVALID_INPUT behavior"
      }
    ],
    "first_call_score": "3/3",
    "first_call_cases": [
      {
        "id": "F3-rerun",
        "ask": "I only have the website vercel.com. Get their LinkedIn company info.",
        "tool": "get_company",
        "args": {
          "identifier": "vercel.com"
        },
        "pass": true,
        "note": "Reaches the company in ONE call, zero extra turns; supersedes the pre-WAVE3 F3 result above."
      },
      {
        "id": "F3b",
        "ask": "Get LinkedIn company info for microsoft.com with a hard 6-credit ceiling.",
        "tool": "get_company",
        "args": {
          "identifier": "microsoft.com",
          "max_credits": 6
        },
        "pass": true,
        "note": "Correctly predicted the domain-resolution quote (8 credits) exceeds max_credits=6 and expected the call to reject uncharged (the honest-quoting description rule)."
      },
      {
        "id": "F3c",
        "ask": "(a LinkedIn URL passed as the company identifier)",
        "tool": "get_company",
        "pass": true,
        "note": "Passed the LinkedIn URL straight through as identifier, without misrouting it through resolve_url first."
      }
    ],
    "source_note": "Raw per-case model transcripts for this rerun are not separately archived in evidence/tool-evals/transcripts.md (only the F6 description-iteration reruns are); this section reproduces the verdicts recorded in evidence/tool-evals/results.md."
  },
  "response_envelopes_used": {
    "note": "Response-usability cases were graded against real production response envelopes captured 2026-07-10. Full envelope bytes are not republished here: they carry a live account's receipt ids and claim token from that snapshot. Envelope shape is documented separately at /openapi.json.",
    "U1": "production get_company identifier=microsoft",
    "U2": "production get_usage",
    "U3": "production search_people keywords=CTO current_company=1035 limit=10"
  },
  "availability_and_latency": {
    "status": "pending",
    "note": "30-day availability and p50/p95 latency are synthetics-derived and require an accumulation window; no data exists yet. Out of scope for this pack (WAVE4 W13); will be added once synthetic monitoring has run for 30 days."
  },
  "verdict": "Descriptions pass picker and first-call at the Haiku floor and Sonnet mid-tier. The one systematic weakness found (a company NAME filter combined with limit=30 on search_people) now has an imperative description rule plus a teaching server error. The WAVE3 F3 domain-emulation rerun is green."
}