{
  "experiment_id": "ticket-router",
  "name": "Ticket router, synthetic regression example",
  "quality_metric": "exact_match",
  "quality_bar": 0.95,
  "objective": "quality",
  "items": [
    {
      "item_id": "case-01",
      "input": "I was charged twice for my last invoice.",
      "reference": "billing"
    },
    {
      "item_id": "case-02",
      "input": "The export button does nothing when I click it.",
      "reference": "technical"
    },
    {
      "item_id": "case-03",
      "input": "How do I change the email address on my account?",
      "reference": "account"
    },
    {
      "item_id": "case-04",
      "input": "My order has not arrived and the tracking has not updated in a week.",
      "reference": "shipping"
    },
    {
      "item_id": "case-05",
      "input": "Why did my subscription price change this month?",
      "reference": "billing"
    },
    {
      "item_id": "case-06",
      "input": "Uploads fail with a 500 error since this morning.",
      "reference": "technical"
    },
    {
      "item_id": "case-07",
      "input": "Please add my colleague as an admin.",
      "reference": "account"
    },
    {
      "item_id": "case-08",
      "input": "The package was delivered to the wrong address.",
      "reference": "shipping"
    },
    {
      "item_id": "case-09",
      "input": "Can I get a receipt for the payment on the 3rd?",
      "reference": "billing"
    },
    {
      "item_id": "case-10",
      "input": "The app crashes when I open a report on my phone.",
      "reference": "technical"
    },
    {
      "item_id": "case-11",
      "input": "I want to delete my account and all my data.",
      "reference": "account"
    },
    {
      "item_id": "case-12",
      "input": "Can I change the delivery address before it ships?",
      "reference": "shipping"
    },
    {
      "item_id": "case-13",
      "input": "The card on file was declined but the money left my bank.",
      "reference": "billing"
    },
    {
      "item_id": "case-14",
      "input": "Search returns no results for exact matches.",
      "reference": "technical"
    },
    {
      "item_id": "case-15",
      "input": "I forgot my password and the reset link never arrives.",
      "reference": "account"
    },
    {
      "item_id": "case-16",
      "input": "The box arrived damaged.",
      "reference": "shipping"
    },
    {
      "item_id": "case-17",
      "input": "I want a refund for the duplicate charge.",
      "reference": "billing"
    },
    {
      "item_id": "case-18",
      "input": "The API returns 401 with a token that worked yesterday.",
      "reference": "technical"
    },
    {
      "item_id": "case-19",
      "input": "Can I merge two accounts into one?",
      "reference": "account"
    },
    {
      "item_id": "case-20",
      "input": "What is the delivery time to Canada?",
      "reference": "shipping"
    },
    {
      "item_id": "case-21",
      "input": "My invoice shows tax for the wrong state.",
      "reference": "billing"
    },
    {
      "item_id": "case-22",
      "input": "Charts do not render in Safari.",
      "reference": "technical"
    },
    {
      "item_id": "case-23",
      "input": "Please change the workspace name.",
      "reference": "account"
    },
    {
      "item_id": "case-24",
      "input": "I received the wrong item.",
      "reference": "shipping"
    },
    {
      "item_id": "case-25",
      "input": "How do I switch from monthly to annual billing?",
      "reference": "billing"
    },
    {
      "item_id": "case-26",
      "input": "The webhook fires twice for one event.",
      "reference": "technical"
    },
    {
      "item_id": "case-27",
      "input": "Remove the former employee from our team.",
      "reference": "account"
    },
    {
      "item_id": "case-28",
      "input": "The tracking number does not work on the carrier site.",
      "reference": "shipping"
    },
    {
      "item_id": "case-29",
      "input": "The promo code did not reduce the total.",
      "reference": "billing"
    },
    {
      "item_id": "case-30",
      "input": "I cannot download the PDF; the link times out.",
      "reference": "technical"
    },
    {
      "item_id": "case-31",
      "input": "How do I transfer ownership of the workspace?",
      "reference": "account"
    },
    {
      "item_id": "case-32",
      "input": "Can you ship to a PO box?",
      "reference": "shipping"
    },
    {
      "item_id": "case-33",
      "input": "Please send the invoice to our accounts payable address.",
      "reference": "billing"
    },
    {
      "item_id": "case-34",
      "input": "Two-factor codes arrive after they expire.",
      "reference": "technical"
    },
    {
      "item_id": "case-35",
      "input": "My username shows my old company name.",
      "reference": "account"
    },
    {
      "item_id": "case-36",
      "input": "One item from the order is missing.",
      "reference": "shipping"
    },
    {
      "item_id": "case-37",
      "input": "Is the trial going to charge my card automatically?",
      "reference": "billing"
    },
    {
      "item_id": "case-38",
      "input": "The integration stopped syncing on Tuesday.",
      "reference": "technical"
    },
    {
      "item_id": "case-39",
      "input": "I need to update the company address on the profile.",
      "reference": "account"
    },
    {
      "item_id": "case-40",
      "input": "How do I return an item that arrived late?",
      "reference": "shipping"
    }
  ],
  "candidates": [
    {
      "id": "reference-v1",
      "label": "Reference run (v1)",
      "configuration": {
        "provider": "your-app",
        "model": "ticket-router-v1",
        "prompt": "Classify the support ticket as billing, technical, account, or shipping. Reply with the label only.",
        "system_prompt": null,
        "params": {
          "temperature": 0
        },
        "source_revision": "v1",
        "endpoint_revision": null
      },
      "outputs": [
        {
          "item_id": "case-01",
          "output": "billing"
        },
        {
          "item_id": "case-02",
          "output": "technical"
        },
        {
          "item_id": "case-03",
          "output": "account"
        },
        {
          "item_id": "case-04",
          "output": "shipping"
        },
        {
          "item_id": "case-05",
          "output": "billing"
        },
        {
          "item_id": "case-06",
          "output": "technical"
        },
        {
          "item_id": "case-07",
          "output": "account"
        },
        {
          "item_id": "case-08",
          "output": "shipping"
        },
        {
          "item_id": "case-09",
          "output": "billing"
        },
        {
          "item_id": "case-10",
          "output": "technical"
        },
        {
          "item_id": "case-11",
          "output": "account"
        },
        {
          "item_id": "case-12",
          "output": "shipping"
        },
        {
          "item_id": "case-13",
          "output": "billing"
        },
        {
          "item_id": "case-14",
          "output": "technical"
        },
        {
          "item_id": "case-15",
          "output": "account"
        },
        {
          "item_id": "case-16",
          "output": "shipping"
        },
        {
          "item_id": "case-17",
          "output": "billing"
        },
        {
          "item_id": "case-18",
          "output": "technical"
        },
        {
          "item_id": "case-19",
          "output": "account"
        },
        {
          "item_id": "case-20",
          "output": "shipping"
        },
        {
          "item_id": "case-21",
          "output": "billing"
        },
        {
          "item_id": "case-22",
          "output": "technical"
        },
        {
          "item_id": "case-23",
          "output": "account"
        },
        {
          "item_id": "case-24",
          "output": "shipping"
        },
        {
          "item_id": "case-25",
          "output": "billing"
        },
        {
          "item_id": "case-26",
          "output": "technical"
        },
        {
          "item_id": "case-27",
          "output": "account"
        },
        {
          "item_id": "case-28",
          "output": "shipping"
        },
        {
          "item_id": "case-29",
          "output": "billing"
        },
        {
          "item_id": "case-30",
          "output": "technical"
        },
        {
          "item_id": "case-31",
          "output": "account"
        },
        {
          "item_id": "case-32",
          "output": "shipping"
        },
        {
          "item_id": "case-33",
          "output": "billing"
        },
        {
          "item_id": "case-34",
          "output": "technical"
        },
        {
          "item_id": "case-35",
          "output": "account"
        },
        {
          "item_id": "case-36",
          "output": "shipping"
        },
        {
          "item_id": "case-37",
          "output": "billing"
        },
        {
          "item_id": "case-38",
          "output": "technical"
        },
        {
          "item_id": "case-39",
          "output": "account"
        },
        {
          "item_id": "case-40",
          "output": "shipping"
        }
      ]
    },
    {
      "id": "candidate-v2",
      "label": "Candidate (v2)",
      "configuration": {
        "provider": "your-app",
        "model": "ticket-router-v2",
        "prompt": "Classify the support ticket as billing, technical, account, or shipping. Reply with the label only.",
        "system_prompt": null,
        "params": {
          "temperature": 0
        },
        "source_revision": "v2",
        "endpoint_revision": null
      },
      "outputs": [
        {
          "item_id": "case-01",
          "output": "billing"
        },
        {
          "item_id": "case-02",
          "output": "technical"
        },
        {
          "item_id": "case-03",
          "output": "account"
        },
        {
          "item_id": "case-04",
          "output": "shipping"
        },
        {
          "item_id": "case-05",
          "output": "billing"
        },
        {
          "item_id": "case-06",
          "output": "technical"
        },
        {
          "item_id": "case-07",
          "output": "billing"
        },
        {
          "item_id": "case-08",
          "output": "shipping"
        },
        {
          "item_id": "case-09",
          "output": "billing"
        },
        {
          "item_id": "case-10",
          "output": "technical"
        },
        {
          "item_id": "case-11",
          "output": "account"
        },
        {
          "item_id": "case-12",
          "output": "shipping"
        },
        {
          "item_id": "case-13",
          "output": "billing"
        },
        {
          "item_id": "case-14",
          "output": "technical"
        },
        {
          "item_id": "case-15",
          "output": "account"
        },
        {
          "item_id": "case-16",
          "output": "shipping"
        },
        {
          "item_id": "case-17",
          "output": "billing"
        },
        {
          "item_id": "case-18",
          "output": "account"
        },
        {
          "item_id": "case-19",
          "output": "account"
        },
        {
          "item_id": "case-20",
          "output": "shipping"
        },
        {
          "item_id": "case-21",
          "output": "billing"
        },
        {
          "item_id": "case-22",
          "output": "technical"
        },
        {
          "item_id": "case-23",
          "output": "account"
        },
        {
          "item_id": "case-24",
          "output": "shipping"
        },
        {
          "item_id": "case-25",
          "output": "technical"
        },
        {
          "item_id": "case-26",
          "output": "technical"
        },
        {
          "item_id": "case-27",
          "output": "account"
        },
        {
          "item_id": "case-28",
          "output": "shipping"
        },
        {
          "item_id": "case-29",
          "output": "billing"
        },
        {
          "item_id": "case-30",
          "output": "technical"
        },
        {
          "item_id": "case-31",
          "output": "account"
        },
        {
          "item_id": "case-32",
          "output": "shipping"
        },
        {
          "item_id": "case-33",
          "output": "billing"
        },
        {
          "item_id": "case-34",
          "output": "technical"
        },
        {
          "item_id": "case-35",
          "output": "account"
        },
        {
          "item_id": "case-36",
          "output": "account"
        },
        {
          "item_id": "case-37",
          "output": "billing"
        },
        {
          "item_id": "case-38",
          "output": "technical"
        },
        {
          "item_id": "case-39",
          "output": "account"
        },
        {
          "item_id": "case-40",
          "output": "shipping"
        }
      ]
    },
    {
      "id": "candidate-v3",
      "label": "Repaired candidate (v3)",
      "configuration": {
        "provider": "your-app",
        "model": "ticket-router-v3",
        "prompt": "Classify the support ticket as billing, technical, account, or shipping. Reply with the label only.",
        "system_prompt": null,
        "params": {
          "temperature": 0
        },
        "source_revision": "v3",
        "endpoint_revision": null
      },
      "outputs": [
        {
          "item_id": "case-01",
          "output": "billing"
        },
        {
          "item_id": "case-02",
          "output": "technical"
        },
        {
          "item_id": "case-03",
          "output": "account"
        },
        {
          "item_id": "case-04",
          "output": "shipping"
        },
        {
          "item_id": "case-05",
          "output": "billing"
        },
        {
          "item_id": "case-06",
          "output": "technical"
        },
        {
          "item_id": "case-07",
          "output": "account"
        },
        {
          "item_id": "case-08",
          "output": "shipping"
        },
        {
          "item_id": "case-09",
          "output": "billing"
        },
        {
          "item_id": "case-10",
          "output": "technical"
        },
        {
          "item_id": "case-11",
          "output": "account"
        },
        {
          "item_id": "case-12",
          "output": "shipping"
        },
        {
          "item_id": "case-13",
          "output": "billing"
        },
        {
          "item_id": "case-14",
          "output": "technical"
        },
        {
          "item_id": "case-15",
          "output": "account"
        },
        {
          "item_id": "case-16",
          "output": "shipping"
        },
        {
          "item_id": "case-17",
          "output": "billing"
        },
        {
          "item_id": "case-18",
          "output": "account"
        },
        {
          "item_id": "case-19",
          "output": "account"
        },
        {
          "item_id": "case-20",
          "output": "shipping"
        },
        {
          "item_id": "case-21",
          "output": "billing"
        },
        {
          "item_id": "case-22",
          "output": "technical"
        },
        {
          "item_id": "case-23",
          "output": "account"
        },
        {
          "item_id": "case-24",
          "output": "shipping"
        },
        {
          "item_id": "case-25",
          "output": "billing"
        },
        {
          "item_id": "case-26",
          "output": "technical"
        },
        {
          "item_id": "case-27",
          "output": "account"
        },
        {
          "item_id": "case-28",
          "output": "shipping"
        },
        {
          "item_id": "case-29",
          "output": "billing"
        },
        {
          "item_id": "case-30",
          "output": "technical"
        },
        {
          "item_id": "case-31",
          "output": "account"
        },
        {
          "item_id": "case-32",
          "output": "shipping"
        },
        {
          "item_id": "case-33",
          "output": "billing"
        },
        {
          "item_id": "case-34",
          "output": "technical"
        },
        {
          "item_id": "case-35",
          "output": "account"
        },
        {
          "item_id": "case-36",
          "output": "shipping"
        },
        {
          "item_id": "case-37",
          "output": "billing"
        },
        {
          "item_id": "case-38",
          "output": "technical"
        },
        {
          "item_id": "case-39",
          "output": "account"
        },
        {
          "item_id": "case-40",
          "output": "shipping"
        }
      ]
    },
    {
      "id": "candidate-v2-partial",
      "label": "Candidate (v2), one case timed out",
      "configuration": {
        "provider": "your-app",
        "model": "ticket-router-v2",
        "prompt": "Classify the support ticket as billing, technical, account, or shipping. Reply with the label only.",
        "system_prompt": null,
        "params": {
          "temperature": 0
        },
        "source_revision": "v2",
        "endpoint_revision": null
      },
      "outputs": [
        {
          "item_id": "case-01",
          "output": "billing"
        },
        {
          "item_id": "case-02",
          "output": "technical"
        },
        {
          "item_id": "case-03",
          "output": "account"
        },
        {
          "item_id": "case-04",
          "output": "shipping"
        },
        {
          "item_id": "case-05",
          "output": "billing"
        },
        {
          "item_id": "case-06",
          "output": "technical"
        },
        {
          "item_id": "case-07",
          "output": "billing"
        },
        {
          "item_id": "case-08",
          "output": "shipping"
        },
        {
          "item_id": "case-09",
          "output": "billing"
        },
        {
          "item_id": "case-10",
          "output": "technical"
        },
        {
          "item_id": "case-11",
          "output": "account"
        },
        {
          "item_id": "case-12",
          "error": "timeout after 30 s"
        },
        {
          "item_id": "case-13",
          "output": "billing"
        },
        {
          "item_id": "case-14",
          "output": "technical"
        },
        {
          "item_id": "case-15",
          "output": "account"
        },
        {
          "item_id": "case-16",
          "output": "shipping"
        },
        {
          "item_id": "case-17",
          "output": "billing"
        },
        {
          "item_id": "case-18",
          "output": "account"
        },
        {
          "item_id": "case-19",
          "output": "account"
        },
        {
          "item_id": "case-20",
          "output": "shipping"
        },
        {
          "item_id": "case-21",
          "output": "billing"
        },
        {
          "item_id": "case-22",
          "output": "technical"
        },
        {
          "item_id": "case-23",
          "output": "account"
        },
        {
          "item_id": "case-24",
          "output": "shipping"
        },
        {
          "item_id": "case-25",
          "output": "technical"
        },
        {
          "item_id": "case-26",
          "output": "technical"
        },
        {
          "item_id": "case-27",
          "output": "account"
        },
        {
          "item_id": "case-28",
          "output": "shipping"
        },
        {
          "item_id": "case-29",
          "output": "billing"
        },
        {
          "item_id": "case-30",
          "output": "technical"
        },
        {
          "item_id": "case-31",
          "output": "account"
        },
        {
          "item_id": "case-32",
          "output": "shipping"
        },
        {
          "item_id": "case-33",
          "output": "billing"
        },
        {
          "item_id": "case-34",
          "output": "technical"
        },
        {
          "item_id": "case-35",
          "output": "account"
        },
        {
          "item_id": "case-36",
          "output": "account"
        },
        {
          "item_id": "case-37",
          "output": "billing"
        },
        {
          "item_id": "case-38",
          "output": "technical"
        },
        {
          "item_id": "case-39",
          "output": "account"
        },
        {
          "item_id": "case-40",
          "output": "shipping"
        }
      ]
    }
  ]
}