Schema and examples (non-Python usage)
This page is for users who want to work directly with the MorganaBench data format (JSON Schema + JSONL examples), especially if they are not using Python (for example, JavaScript/TypeScript, Go, or Java).
The SDK ships:
A JSON Schema for the benchmark
ExampleobjectExample benchmark JSONL files (unexecuted and executed)
These files are generated by the SDK and are intended to be consumed by tools outside of Python as well.
JSON Schema (Example)
{
"$defs": {
"ChatMessage": {
"description": "An OpenAI-style message in the conversation.",
"properties": {
"role": {
"title": "Role",
"type": "string"
},
"content": {
"title": "Content",
"type": "string"
}
},
"required": [
"role",
"content"
],
"title": "ChatMessage",
"type": "object"
},
"Citation": {
"properties": {
"document_id": {
"title": "Document Id",
"type": "string"
},
"span_from": {
"title": "Span From",
"type": "integer"
},
"span_to": {
"title": "Span To",
"type": "integer"
}
},
"required": [
"document_id",
"span_from",
"span_to"
],
"title": "Citation",
"type": "object"
},
"DateTimeMatcher": {
"description": "Matches an argument if it matches a given date and / or time.\n\nThe match is determined as semantic equivalence, given the actual argument value, and the request time as provided\nin the output environment.",
"properties": {
"match_as": {
"const": "date_time",
"default": "date_time",
"title": "Match As",
"type": "string"
},
"value": {
"title": "Value",
"type": "string"
}
},
"required": [
"value"
],
"title": "DateTimeMatcher",
"type": "object"
},
"EmailMatcher": {
"description": "Matches an argument if it matches the given email address.",
"properties": {
"match_as": {
"const": "email",
"default": "email",
"title": "Match As",
"type": "string"
},
"value": {
"title": "Value",
"type": "string"
}
},
"required": [
"value"
],
"title": "EmailMatcher",
"type": "object"
},
"Environment": {
"additionalProperties": true,
"properties": {
"user_time": {
"anyOf": [
{
"format": "date-time",
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "User Time"
}
},
"title": "Environment",
"type": "object"
},
"EqualsMatcher": {
"description": "Matches an argument if it equals a given value.",
"properties": {
"match_as": {
"const": "equality",
"default": "equality",
"title": "Match As",
"type": "string"
},
"value": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
},
{
"type": "number"
},
{
"type": "boolean"
}
],
"title": "Value"
}
},
"required": [
"value"
],
"title": "EqualsMatcher",
"type": "object"
},
"Expectations": {
"additionalProperties": true,
"description": "Agent expectations description.",
"properties": {
"expected_response": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Expected Response"
},
"assertions": {
"items": {
"discriminator": {
"mapping": {
"no_tool_called": "#/$defs/NoToolCallAssertion",
"tool_called": "#/$defs/ToolCallAssertion"
},
"propertyName": "assert_that"
},
"oneOf": [
{
"$ref": "#/$defs/ToolCallAssertion"
},
{
"$ref": "#/$defs/NoToolCallAssertion"
}
]
},
"title": "Assertions",
"type": "array"
}
},
"title": "Expectations",
"type": "object"
},
"FreeTextMatcher": {
"description": "Matches an argument if semantically it achieves the same goal as the given free text.",
"properties": {
"match_as": {
"const": "free_text",
"default": "free_text",
"title": "Match As",
"type": "string"
},
"value": {
"title": "Value",
"type": "string"
}
},
"required": [
"value"
],
"title": "FreeTextMatcher",
"type": "object"
},
"InputMetadata": {
"properties": {
"turns": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/TurnMetadata"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"title": "Turns"
},
"categories": {
"anyOf": [
{
"additionalProperties": {
"type": "string"
},
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"title": "Categories"
}
},
"title": "InputMetadata",
"type": "object"
},
"Inputs": {
"additionalProperties": true,
"description": "Agent input description.",
"properties": {
"messages": {
"items": {
"$ref": "#/$defs/ChatMessage"
},
"minItems": 1,
"title": "Messages",
"type": "array"
},
"metadata": {
"anyOf": [
{
"$ref": "#/$defs/InputMetadata"
},
{
"type": "null"
}
],
"default": null
},
"tools": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"title": "Tools"
}
},
"required": [
"messages"
],
"title": "Inputs",
"type": "object"
},
"JsonValue": {},
"MissingMatcher": {
"description": "Matches an argument if it is missing (not provided).",
"properties": {
"match_as": {
"const": "missing",
"default": "missing",
"title": "Match As",
"type": "string"
}
},
"title": "MissingMatcher",
"type": "object"
},
"NoToolCallAssertion": {
"description": "Assert that no tool call was made.",
"properties": {
"assert_that": {
"const": "no_tool_called",
"default": "no_tool_called",
"title": "Assert That",
"type": "string"
}
},
"title": "NoToolCallAssertion",
"type": "object"
},
"OneParameterAssertion": {
"description": "Passes when a single argument matches the given matcher.",
"properties": {
"param": {
"title": "Param",
"type": "string"
},
"matcher": {
"discriminator": {
"mapping": {
"date_time": "#/$defs/DateTimeMatcher",
"email": "#/$defs/EmailMatcher",
"equality": "#/$defs/EqualsMatcher",
"free_text": "#/$defs/FreeTextMatcher",
"missing": "#/$defs/MissingMatcher",
"optional": "#/$defs/OptionalMatcher"
},
"propertyName": "match_as"
},
"oneOf": [
{
"$ref": "#/$defs/EqualsMatcher"
},
{
"$ref": "#/$defs/FreeTextMatcher"
},
{
"$ref": "#/$defs/DateTimeMatcher"
},
{
"$ref": "#/$defs/EmailMatcher"
},
{
"$ref": "#/$defs/MissingMatcher"
},
{
"$ref": "#/$defs/OptionalMatcher"
}
],
"title": "Matcher"
}
},
"required": [
"param",
"matcher"
],
"title": "OneParameterAssertion",
"type": "object"
},
"OptionalMatcher": {
"description": "Matches an argument either if it is missing (not provided) or its value matches the given matcher.",
"properties": {
"match_as": {
"const": "optional",
"default": "optional",
"title": "Match As",
"type": "string"
},
"default": {
"$ref": "#/$defs/ValueMatcher"
}
},
"required": [
"default"
],
"title": "OptionalMatcher",
"type": "object"
},
"OptionalMatcher_Annotated_Union_FreeTextMatcher__DateTimeMatcher___FieldInfo_annotation_NoneType__required_True__discriminator__match_as____": {
"properties": {
"match_as": {
"const": "optional",
"default": "optional",
"title": "Match As",
"type": "string"
},
"default": {
"oneOf": [
{
"$ref": "#/$defs/FreeTextMatcher"
},
{
"$ref": "#/$defs/DateTimeMatcher"
}
],
"discriminator": {
"mapping": {
"date_time": "#/$defs/DateTimeMatcher",
"free_text": "#/$defs/FreeTextMatcher"
},
"propertyName": "match_as"
},
"title": "Default"
}
},
"required": [
"default"
],
"title": "OptionalMatcher[Annotated[Union[FreeTextMatcher, DateTimeMatcher], FieldInfo(annotation=NoneType, required=True, discriminator='match_as')]]",
"type": "object"
},
"Outputs": {
"additionalProperties": true,
"properties": {
"response": {
"title": "Response",
"type": "string"
},
"citations": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/Citation"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"title": "Citations"
},
"environment": {
"anyOf": [
{
"$ref": "#/$defs/Environment"
},
{
"type": "null"
}
],
"default": null
},
"trace": {
"default": [],
"items": {
"discriminator": {
"mapping": {
"retriever": "#/$defs/RetrievalResults",
"tool_call": "#/$defs/ToolCall",
"tool_result": "#/$defs/ToolResult"
},
"propertyName": "event"
},
"oneOf": [
{
"$ref": "#/$defs/ToolCall"
},
{
"$ref": "#/$defs/ToolResult"
},
{
"$ref": "#/$defs/RetrievalResults"
}
]
},
"title": "Trace",
"type": "array"
}
},
"required": [
"response"
],
"title": "Outputs",
"type": "object"
},
"ParameterGroupAssertion": {
"description": "Passes when a group of arguments matches the given matcher.",
"properties": {
"params": {
"items": {
"type": "string"
},
"title": "Params",
"type": "array"
},
"matcher": {
"discriminator": {
"mapping": {
"date_time": "#/$defs/DateTimeMatcher",
"free_text": "#/$defs/FreeTextMatcher",
"optional": "#/$defs/OptionalMatcher_Annotated_Union_FreeTextMatcher__DateTimeMatcher___FieldInfo_annotation_NoneType__required_True__discriminator__match_as____"
},
"propertyName": "match_as"
},
"oneOf": [
{
"$ref": "#/$defs/FreeTextMatcher"
},
{
"$ref": "#/$defs/DateTimeMatcher"
},
{
"$ref": "#/$defs/OptionalMatcher_Annotated_Union_FreeTextMatcher__DateTimeMatcher___FieldInfo_annotation_NoneType__required_True__discriminator__match_as____"
}
],
"title": "Matcher"
}
},
"required": [
"params",
"matcher"
],
"title": "ParameterGroupAssertion",
"type": "object"
},
"RetrievalResult": {
"additionalProperties": true,
"properties": {
"id": {
"title": "Id",
"type": "string"
},
"page_content": {
"title": "Page Content",
"type": "string"
}
},
"required": [
"id",
"page_content"
],
"title": "RetrievalResult",
"type": "object"
},
"RetrievalResults": {
"additionalProperties": true,
"properties": {
"event": {
"const": "retriever",
"default": "retriever",
"title": "Event",
"type": "string"
},
"outputs": {
"items": {
"$ref": "#/$defs/RetrievalResult"
},
"title": "Outputs",
"type": "array"
}
},
"required": [
"outputs"
],
"title": "RetrievalResults",
"type": "object"
},
"ToolCall": {
"properties": {
"event": {
"const": "tool_call",
"default": "tool_call",
"title": "Event",
"type": "string"
},
"id": {
"title": "Id",
"type": "string"
},
"tool": {
"title": "Tool",
"type": "string"
},
"params": {
"additionalProperties": {
"$ref": "#/$defs/JsonValue"
},
"title": "Params",
"type": "object"
}
},
"required": [
"id",
"tool",
"params"
],
"title": "ToolCall",
"type": "object"
},
"ToolCallAssertion": {
"description": "Assert that a tool call was made with given parameters.",
"properties": {
"assert_that": {
"const": "tool_called",
"default": "tool_called",
"title": "Assert That",
"type": "string"
},
"tool": {
"title": "Tool",
"type": "string"
},
"parameters": {
"default": [],
"items": {
"oneOf": [
{
"$ref": "#/$defs/OneParameterAssertion"
},
{
"$ref": "#/$defs/ParameterGroupAssertion"
}
]
},
"title": "Parameters",
"type": "array"
}
},
"required": [
"tool"
],
"title": "ToolCallAssertion",
"type": "object"
},
"ToolResult": {
"properties": {
"event": {
"const": "tool_result",
"default": "tool_result",
"title": "Event",
"type": "string"
},
"id": {
"title": "Id",
"type": "string"
},
"result": {
"$ref": "#/$defs/JsonValue"
}
},
"required": [
"id",
"result"
],
"title": "ToolResult",
"type": "object"
},
"TurnMetadata": {
"properties": {
"categories": {
"additionalProperties": {
"type": "string"
},
"title": "Categories",
"type": "object"
},
"resources": {
"items": {
"$ref": "#/$defs/JsonValue"
},
"title": "Resources",
"type": "array"
}
},
"required": [
"categories",
"resources"
],
"title": "TurnMetadata",
"type": "object"
},
"ValueMatcher": {
"properties": {},
"title": "ValueMatcher",
"type": "object"
}
},
"additionalProperties": true,
"description": "A benchmark (eval dataset) example.",
"properties": {
"inputs": {
"$ref": "#/$defs/Inputs"
},
"expectations": {
"$ref": "#/$defs/Expectations"
},
"outputs": {
"anyOf": [
{
"$ref": "#/$defs/Outputs"
},
{
"type": "null"
}
],
"default": null
}
},
"required": [
"inputs",
"expectations"
],
"title": "Example",
"type": "object"
}
Example benchmark JSONL (unexecuted)
Each line is a single JSON object representing one Example (with inputs + expectations, and typically no outputs).
{"inputs": {"messages": [{"role": "user", "content": "Who is the King of England?"}], "tools": ["search"]}, "expectations": {"expected_response": "King Charles III is the current monarch of the United Kingdom.", "assertions": [{"assert_that": "tool_called", "tool": "search", "parameters": [{"param": "query", "matcher": {"match_as": "free_text", "value": "King Charles III"}}, {"param": "limit", "matcher": {"match_as": "equality", "value": 5}}, {"param": "site", "matcher": {"match_as": "missing"}}]}]}}
{"inputs": {"messages": [{"role": "user", "content": "Schedule a design review next Friday at 2pm with Alex at HQ."}], "metadata": {"turns": [{"categories": {"intent": "calendar_event"}, "resources": ["https://calendar.example.com/help", {"doc_id": "policy-42", "path": "docs/policy-42.pdf"}]}], "categories": {"topic": "scheduling", "priority": "normal"}}, "tools": ["calendar"]}, "expectations": {"expected_response": "Sure, I will schedule the design review.", "assertions": [{"assert_that": "tool_called", "tool": "calendar", "parameters": [{"params": ["start_time", "end_time"], "matcher": {"match_as": "date_time", "value": "next Friday at 2pm"}}, {"params": ["title", "location"], "matcher": {"match_as": "optional", "default": {"match_as": "free_text", "value": "Design review"}}}, {"param": "attendees", "matcher": {"match_as": "missing"}}]}]}}
{"inputs": {"messages": [{"role": "user", "content": "What is the capital of France?"}]}, "expectations": {"expected_response": "Paris is the capital of France.", "assertions": []}}
Example executed benchmark JSONL
Each line is a single JSON object representing one executed Example (with inputs + expectations + outputs).
{"inputs": {"messages": [{"role": "user", "content": "Who is the King of England?"}], "tools": ["search"]}, "expectations": {"expected_response": "King Charles III is the current monarch of the United Kingdom.", "assertions": [{"assert_that": "tool_called", "tool": "search", "parameters": [{"param": "query", "matcher": {"match_as": "free_text", "value": "King Charles III"}}, {"param": "limit", "matcher": {"match_as": "equality", "value": 5}}, {"param": "site", "matcher": {"match_as": "missing"}}]}]}, "outputs": {"response": "King Charles III is the current monarch of the United Kingdom.", "trace": [{"event": "tool_call", "id": "call_1", "tool": "search", "params": {"query": "King Charles III", "limit": 5}}, {"event": "tool_result", "id": "call_1", "result": {"status": "ok", "items": ["https://en.wikipedia.org/wiki/King_Charles_III"]}}]}}
{"inputs": {"messages": [{"role": "user", "content": "Schedule a design review next Friday at 2pm with Alex at HQ."}], "metadata": {"turns": [{"categories": {"intent": "calendar_event"}, "resources": ["https://calendar.example.com/help", {"doc_id": "policy-42", "path": "docs/policy-42.pdf"}]}], "categories": {"topic": "scheduling", "priority": "normal"}}, "tools": ["calendar"]}, "expectations": {"expected_response": "Sure, I will schedule the design review.", "assertions": [{"assert_that": "tool_called", "tool": "calendar", "parameters": [{"params": ["start_time", "end_time"], "matcher": {"match_as": "date_time", "value": "next Friday at 2pm"}}, {"params": ["title", "location"], "matcher": {"match_as": "optional", "default": {"match_as": "free_text", "value": "Design review"}}}, {"param": "attendees", "matcher": {"match_as": "missing"}}]}]}, "outputs": {"response": "I've scheduled the design review for next Friday at 2pm.", "environment": {"user_time": "2024-02-01T09:15:00"}, "trace": [{"event": "tool_call", "id": "call_2", "tool": "calendar", "params": {"start_time": "2024-02-09T14:00:00", "end_time": "2024-02-09T15:00:00", "title": "Design review", "location": "HQ"}}, {"event": "tool_result", "id": "call_2", "result": {"status": "created", "event_id": "evt_123", "calendar": "primary"}}]}}
{"inputs": {"messages": [{"role": "user", "content": "What is the capital of France?"}]}, "expectations": {"expected_response": "Paris is the capital of France.", "assertions": []}, "outputs": {"response": "Paris is the capital of France.", "citations": [{"document_id": "doc_1", "span_from": 0, "span_to": 53}, {"document_id": "doc_2", "span_from": 0, "span_to": 30}], "trace": [{"event": "retriever", "outputs": [{"id": "doc_1", "page_content": "Paris is the capital and most populous city of France."}, {"id": "doc_2", "page_content": "France's capital city is Paris."}]}]}}