{ "openapi": "3.1.0", "info": { "title": "PandaProbe API", "description": "Open-source agent engineering platform \u2014 trace LLM calls, evaluate agent performance, and monitor AI pipelines.", "version": "0.7.0", "contact": { "name": "PandaProbe", "url": "https://www.pandaprobe.com", "email": "support@pandaprobe.com" } }, "paths": { "/traces": { "post": { "tags": [ "traces" ], "summary": "Ingest Trace", "description": "Accept a trace payload for asynchronous persistence (upsert).\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`\n\nRate limit: `100/min`", "operationId": "ingest_trace_traces_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceCreate" } } } }, "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceAccepted" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "traces" ], "summary": "List Traces", "description": "List traces for the current project with filtering, sorting, and stats.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "list_traces_traces_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } }, { "name": "session_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id" } }, { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/TraceStatus" }, { "type": "null" } ], "title": "Status" } }, { "name": "user_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" } }, { "name": "tags", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "title": "Tags" } }, { "name": "name", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" } }, { "name": "started_after", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Started After" } }, { "name": "started_before", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Started Before" } }, { "name": "sort_by", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/TraceSortBy", "default": "started_at" } }, { "name": "sort_order", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SortOrder", "default": "desc" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_TraceListItem_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/traces/analytics": { "get": { "tags": [ "traces" ], "summary": "Get Analytics", "description": "Time-series analytics for traces.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "get_analytics_traces_analytics_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/AnalyticsMetric", "default": "volume" } }, { "name": "granularity", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/AnalyticsGranularity", "default": "day" } }, { "name": "started_after", "in": "query", "required": true, "schema": { "type": "string", "format": "date-time", "title": "Started After" } }, { "name": "started_before", "in": "query", "required": true, "schema": { "type": "string", "format": "date-time", "title": "Started Before" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "type": "array", "items": { "$ref": "#/components/schemas/AnalyticsBucket" } }, { "type": "array", "items": { "$ref": "#/components/schemas/TokenCostBucket" } }, { "type": "array", "items": { "$ref": "#/components/schemas/TopModel" } } ], "title": "Response Get Analytics Traces Analytics Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/traces/users": { "get": { "tags": [ "traces" ], "summary": "List Trace Users", "description": "List unique user_ids with trace statistics.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "list_trace_users_traces_users_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_UserSummary_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/traces/batch/delete": { "post": { "tags": [ "traces" ], "summary": "Batch Delete", "description": "Delete multiple traces at once.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "batch_delete_traces_batch_delete_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchDeleteRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchDeleteResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/traces/batch/tags": { "post": { "tags": [ "traces" ], "summary": "Batch Tags", "description": "Add or remove tags on multiple traces.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "batch_tags_traces_batch_tags_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchTagsRequest" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BatchTagsResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/traces/{trace_id}": { "patch": { "tags": [ "traces" ], "summary": "Update Trace", "description": "Partially update a trace.\n\nOnly fields present in the request body are touched. Sending\n``\"session_id\": null`` explicitly clears the field; omitting\n``session_id`` entirely leaves it unchanged. ``metadata`` is\nshallow-merged with existing values.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "update_trace_traces__trace_id__patch", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceUpdate" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "traces" ], "summary": "Get Trace", "description": "Retrieve a single trace with all its spans.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "get_trace_traces__trace_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "traces" ], "summary": "Delete Trace", "description": "Delete a trace and all its spans.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "delete_trace_traces__trace_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/traces/{trace_id}/spans": { "post": { "tags": [ "traces" ], "summary": "Add Spans", "description": "Add one or more spans to an existing trace (upsert).\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "add_spans_traces__trace_id__spans_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/SpanCreate" }, "minItems": 1, "maxItems": 500, "title": "Body" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SpansAccepted" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/traces/{trace_id}/spans/{span_id}": { "patch": { "tags": [ "traces" ], "summary": "Update Span", "description": "Partially update a span on a trace.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "update_span_traces__trace_id__spans__span_id__patch", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } }, { "name": "span_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Span Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SpanUpdate" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SpanResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/sessions": { "get": { "tags": [ "sessions" ], "summary": "List Sessions", "description": "List sessions for the current project.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "list_sessions_sessions_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } }, { "name": "user_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" } }, { "name": "has_error", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Has Error" } }, { "name": "started_after", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Started After" } }, { "name": "started_before", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Started Before" } }, { "name": "tags", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "array", "items": { "type": "string" } }, { "type": "null" } ], "title": "Tags" } }, { "name": "query", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "ILIKE filter on session_id", "title": "Query" }, "description": "ILIKE filter on session_id" }, { "name": "sort_by", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SessionSortBy", "default": "recent" } }, { "name": "sort_order", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/SortOrder", "default": "desc" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_SessionSummary_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/sessions/analytics": { "get": { "tags": [ "sessions" ], "summary": "Get Session Analytics", "description": "Time-series session statistics.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "get_session_analytics_sessions_analytics_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "granularity", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/AnalyticsGranularity", "default": "day" } }, { "name": "started_after", "in": "query", "required": true, "schema": { "type": "string", "format": "date-time", "title": "Started After" } }, { "name": "started_before", "in": "query", "required": true, "schema": { "type": "string", "format": "date-time", "title": "Started Before" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/SessionAnalyticsBucket" }, "title": "Response Get Session Analytics Sessions Analytics Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/sessions/{session_id}": { "get": { "tags": [ "sessions" ], "summary": "Get Session", "description": "Retrieve a single session with its full traces (including spans).\n\nSummary fields (trace_count, latency, error, tags, tokens, cost)\nare computed via SQL aggregation over *all* traces in the session,\nregardless of the pagination window.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "get_session_sessions__session_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "session_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Session Id" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 1000, "minimum": 1, "default": 200, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionDetail" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "sessions" ], "summary": "Delete Session", "description": "Delete all traces (and cascaded spans) for a session.\n\nAuth: `Bearer` + `X-Project-ID` | `X-API-Key` + `X-Project-Name`", "operationId": "delete_session_sessions__session_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "session_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Session Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionDeleteResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/providers": { "get": { "tags": [ "evaluations" ], "summary": "Get Available Providers", "description": "List LLM providers and their availability.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_available_providers_evaluations_providers_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/ProviderInfo" }, "type": "array", "title": "Response Get Available Providers Evaluations Providers Get" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/evaluations/trace-metrics": { "get": { "tags": [ "evaluations" ], "summary": "Get Available Metrics", "description": "List all registered evaluation metrics.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_available_metrics_evaluations_trace_metrics_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/MetricSummary" }, "type": "array", "title": "Response Get Available Metrics Evaluations Trace Metrics Get" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/evaluations/trace-runs/template": { "get": { "tags": [ "evaluations" ], "summary": "Get Eval Run Template", "description": "Return a pre-filled eval run template for a single metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_eval_run_template_evaluations_trace_runs_template_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric", "in": "query", "required": true, "schema": { "type": "string", "description": "Metric name to build the template for", "title": "Metric" }, "description": "Metric name to build the template for" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunTemplate" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-runs": { "post": { "tags": [ "evaluations" ], "summary": "Create Eval Run", "description": "Create a filtered eval run.\n\nResolves traces matching the provided filters, optionally samples\na fraction of them, then dispatches a background Celery task to\nrun the requested metrics asynchronously via an LLM judge.\n\n**Request body fields:**\n\n- **name** *(string, optional)*: Human-readable label, e.g. ``\"Weekly prod eval\"``.\n- **metrics** *(string[], required)*: Metric names to run. Get available names\n from ``GET /evaluations/trace-metrics``. Example: ``[\"task_completion\", \"tool_correctness\"]``.\n- **filters** *(object, optional)*: Trace selection filters. All fields optional:\n - **date_from**: ISO 8601 datetime, e.g. ``\"2025-01-15T00:00:00Z\"``.\n Includes traces started on or after this time.\n - **date_to**: ISO 8601 datetime. Includes traces started before this time (exclusive).\n - **status**: One of ``PENDING``, ``RUNNING``, ``COMPLETED``, ``ERROR``.\n - **session_id**: Exact session ID string.\n - **user_id**: Exact user ID string.\n - **tags**: Array of strings, e.g. ``[\"production\", \"v2\"]``. Matches traces with ANY tag.\n - **name**: Substring match on trace name (case-insensitive).\n- **sampling_rate** *(float, optional, default 1.0)*: Fraction of matching traces\n to evaluate. ``1.0`` = all, ``0.1`` = random 10%.\n- **model** *(string, optional)*: LLM model override, e.g. ``\"openai/gpt-4o\"``.\n Uses system default if null.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "create_eval_run_evaluations_trace_runs_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateEvalRunRequest" } } } }, "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "evaluations" ], "summary": "List Eval Runs", "description": "List trace eval runs (summary view).\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_eval_runs_evaluations_trace_runs_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/EvaluationStatus" }, { "type": "null" } ], "title": "Status" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_EvalRunResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-runs/batch": { "post": { "tags": [ "evaluations" ], "summary": "Create Batch Eval Run", "description": "Create an eval run for an explicit list of trace IDs.\n\nEvaluates exactly the provided traces with all requested metrics.\nAll metrics for all traces are processed in a single sequential\nCelery task -- no race conditions on concurrent writes.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "create_batch_eval_run_evaluations_trace_runs_batch_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateBatchEvalRunRequest" } } }, "required": true }, "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/evaluations/trace-runs/{run_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Eval Run", "description": "Get full eval run detail.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_eval_run_evaluations_trace_runs__run_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "evaluations" ], "summary": "Delete Eval Run", "description": "Delete an eval run.\n\nBy default, only the run record is deleted (scores are preserved\nwith ``eval_run_id`` set to NULL via CASCADE). Pass\n``?delete_scores=true`` to also delete all scores from this run.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "delete_eval_run_evaluations_trace_runs__run_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } }, { "name": "delete_scores", "in": "query", "required": false, "schema": { "type": "boolean", "description": "Also delete all trace scores created by this run.", "default": false, "title": "Delete Scores" }, "description": "Also delete all trace scores created by this run." } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-runs/{run_id}/retry": { "post": { "tags": [ "evaluations" ], "summary": "Retry Failed Eval Run", "description": "Retry failed metrics from a completed eval run.\n\nCreates a new eval run targeting only the trace+metric pairs that\nfailed in the original run. Returns 422 if the original run has\nno failures to retry.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "retry_failed_eval_run_evaluations_trace_runs__run_id__retry_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-runs/{run_id}/scores": { "get": { "tags": [ "evaluations" ], "summary": "Get Scores For Run", "description": "List all trace scores produced by a specific eval run.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_scores_for_run_evaluations_trace_runs__run_id__scores_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/TraceScoreResponse" }, "title": "Response Get Scores For Run Evaluations Trace Runs Run Id Scores Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-scores": { "post": { "tags": [ "evaluations" ], "summary": "Create Trace Score", "description": "Manually create a trace score.\n\nUse ``source=ANNOTATION`` for human-created scores from the dashboard,\nor ``source=PROGRAMMATIC`` for SDK/API-submitted scores.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "create_trace_score_evaluations_trace_scores_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateTraceScoreRequest" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceScoreResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "evaluations" ], "summary": "List Trace Scores", "description": "List trace scores (summary view) with comprehensive filters.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_trace_scores_evaluations_trace_scores_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "description": "Filter by trace UUID", "title": "Trace Id" }, "description": "Filter by trace UUID" }, { "name": "name", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Filter by metric name (exact match)", "title": "Name" }, "description": "Filter by metric name (exact match)" }, { "name": "source", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/ScoreSource" }, { "type": "null" } ], "description": "Filter by score source: AUTOMATED, ANNOTATION, PROGRAMMATIC", "title": "Source" }, "description": "Filter by score source: AUTOMATED, ANNOTATION, PROGRAMMATIC" }, { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/ScoreStatus" }, { "type": "null" } ], "description": "Filter by score status: SUCCESS, FAILED, PENDING", "title": "Status" }, "description": "Filter by score status: SUCCESS, FAILED, PENDING" }, { "name": "data_type", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/ScoreDataType" }, { "type": "null" } ], "description": "Filter by data type: NUMERIC, BOOLEAN, CATEGORICAL", "title": "Data Type" }, "description": "Filter by data type: NUMERIC, BOOLEAN, CATEGORICAL" }, { "name": "eval_run_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "description": "Filter by eval run UUID", "title": "Eval Run Id" }, "description": "Filter by eval run UUID" }, { "name": "environment", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Filter by trace environment (exact match)", "title": "Environment" }, "description": "Filter by trace environment (exact match)" }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime. Include scores created on or after.", "title": "Date From" }, "description": "ISO 8601 datetime. Include scores created on or after." }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime. Include scores created before (exclusive).", "title": "Date To" }, "description": "ISO 8601 datetime. Include scores created before (exclusive)." }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "description": "Page size", "default": 50, "title": "Limit" }, "description": "Page size" }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "description": "Number of items to skip", "default": 0, "title": "Offset" }, "description": "Number of items to skip" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_TraceScoreResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-scores/{trace_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Scores For Trace", "description": "Get the latest score per metric for a specific trace.\n\nReturns one score per metric name, deduplicated by most recent\n``created_at`` regardless of status. The dashboard uses this to\ndisplay a score overview panel for the trace.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_scores_for_trace_evaluations_trace_scores__trace_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/TraceScoreResponse" }, "title": "Response Get Scores For Trace Evaluations Trace Scores Trace Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/trace-scores/{score_id}": { "patch": { "tags": [ "evaluations" ], "summary": "Update Trace Score", "description": "Manually edit a trace score.\n\nOnly ``value``, ``reason``, and ``metadata`` can be changed by the\ncaller. ``status`` is automatically set to SUCCESS, ``source`` is\nchanged to ANNOTATION, and ``updated_at`` is set to now.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "update_trace_score_evaluations_trace_scores__score_id__patch", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "score_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Score Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateTraceScoreRequest" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceScoreResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "evaluations" ], "summary": "Delete Trace Score", "description": "Delete a single trace score.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "delete_trace_score_evaluations_trace_scores__score_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "score_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Score Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/trace-scores/summary": { "get": { "tags": [ "evaluations" ], "summary": "Get Trace Score Analytics Summary", "description": "Aggregated trace score summary per metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_trace_score_analytics_summary_evaluations_analytics_trace_scores_summary_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime. Include scores created on or after.", "title": "Date From" }, "description": "ISO 8601 datetime. Include scores created on or after." }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime. Include scores created before (exclusive).", "title": "Date To" }, "description": "ISO 8601 datetime. Include scores created before (exclusive)." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreSummaryItem" }, "title": "Response Get Trace Score Analytics Summary Evaluations Analytics Trace Scores Summary Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/trace-scores/trend": { "get": { "tags": [ "evaluations" ], "summary": "Get Trace Score Analytics Trend", "description": "Time series of average trace scores by metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_trace_score_analytics_trend_evaluations_analytics_trace_scores_trend_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", "description": "Metric name to get trend for", "title": "Metric Name" }, "description": "Metric name to get trend for" }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date From" }, "description": "ISO 8601 datetime." }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date To" }, "description": "ISO 8601 datetime." }, { "name": "granularity", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/AnalyticsGranularity", "description": "Time bucket: hour, day, week", "default": "day" }, "description": "Time bucket: hour, day, week" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreTrendItem" }, "title": "Response Get Trace Score Analytics Trend Evaluations Analytics Trace Scores Trend Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/trace-scores/distribution": { "get": { "tags": [ "evaluations" ], "summary": "Get Trace Score Analytics Distribution", "description": "Histogram of trace score values for a metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_trace_score_analytics_distribution_evaluations_analytics_trace_scores_distribution_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", "description": "Metric name to get distribution for", "title": "Metric Name" }, "description": "Metric name to get distribution for" }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date From" }, "description": "ISO 8601 datetime." }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date To" }, "description": "ISO 8601 datetime." }, { "name": "buckets", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, "description": "Number of histogram buckets (1-100)", "default": 10, "title": "Buckets" }, "description": "Number of histogram buckets (1-100)" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreDistributionItem" }, "title": "Response Get Trace Score Analytics Distribution Evaluations Analytics Trace Scores Distribution Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-metrics": { "get": { "tags": [ "evaluations" ], "summary": "Get Available Session Metrics", "description": "List all registered session evaluation metrics.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_available_session_metrics_evaluations_session_metrics_get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/MetricSummary" }, "type": "array", "title": "Response Get Available Session Metrics Evaluations Session Metrics Get" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/evaluations/session-runs": { "post": { "tags": [ "evaluations" ], "summary": "Create Session Eval Run", "description": "Create a filter-based session eval run.\n\nResolves sessions matching the provided filters, then dispatches a\nbackground Celery task that computes trace-level signals and\naggregates them into session-level metrics.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "create_session_eval_run_evaluations_session_runs_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateSessionEvalRunRequest" } } } }, "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "evaluations" ], "summary": "List Session Eval Runs", "description": "List session eval runs.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_session_eval_runs_evaluations_session_runs_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/EvaluationStatus" }, { "type": "null" } ], "title": "Status" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_EvalRunResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-runs/batch": { "post": { "tags": [ "evaluations" ], "summary": "Create Batch Session Eval Run", "description": "Create a session eval run for explicit session IDs.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "create_batch_session_eval_run_evaluations_session_runs_batch_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateBatchSessionEvalRunRequest" } } }, "required": true }, "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ] } }, "/evaluations/session-runs/{run_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Eval Run", "description": "Get full session eval run detail.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_eval_run_evaluations_session_runs__run_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "evaluations" ], "summary": "Delete Session Eval Run", "description": "Delete a session eval run.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "delete_session_eval_run_evaluations_session_runs__run_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } }, { "name": "delete_scores", "in": "query", "required": false, "schema": { "type": "boolean", "description": "Also delete all session scores from this run.", "default": false, "title": "Delete Scores" }, "description": "Also delete all session scores from this run." } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-runs/{run_id}/retry": { "post": { "tags": [ "evaluations" ], "summary": "Retry Failed Session Eval Run", "description": "Retry failed metrics from a completed session eval run.\n\nCreates a new session eval run targeting only the session+metric\npairs that failed in the original run. Returns 422 if the original\nrun has no failures to retry.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``50/min``", "operationId": "retry_failed_session_eval_run_evaluations_session_runs__run_id__retry_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-runs/{run_id}/scores": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Scores For Run", "description": "List all session scores produced by a specific eval run.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_scores_for_run_evaluations_session_runs__run_id__scores_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "run_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Run Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/SessionScoreResponse" }, "title": "Response Get Session Scores For Run Evaluations Session Runs Run Id Scores Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-scores": { "get": { "tags": [ "evaluations" ], "summary": "List Session Scores", "description": "List session scores with filters.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_session_scores_evaluations_session_scores_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "session_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Filter by session ID", "title": "Session Id" }, "description": "Filter by session ID" }, { "name": "name", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Filter by metric name", "title": "Name" }, "description": "Filter by metric name" }, { "name": "source", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/ScoreSource" }, { "type": "null" } ], "title": "Source" } }, { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/ScoreStatus" }, { "type": "null" } ], "title": "Status" } }, { "name": "eval_run_id", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Eval Run Id" } }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date From" } }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date To" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_SessionScoreResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-scores/{session_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Scores For Session", "description": "Get all scores for a specific session.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_scores_for_session_evaluations_session_scores__session_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "session_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Session Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/SessionScoreResponse" }, "title": "Response Get Scores For Session Evaluations Session Scores Session Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/session-scores/{score_id}": { "delete": { "tags": [ "evaluations" ], "summary": "Delete Session Score", "description": "Delete a single session score.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "delete_session_score_evaluations_session_scores__score_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "score_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Score Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/session-scores/summary": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Score Analytics Summary", "description": "Aggregated session score summary per metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_score_analytics_summary_evaluations_analytics_session_scores_summary_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date From" } }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date To" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreSummaryItem" }, "title": "Response Get Session Score Analytics Summary Evaluations Analytics Session Scores Summary Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/session-scores/trend": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Score Analytics Trend", "description": "Time series of average session scores by metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_score_analytics_trend_evaluations_analytics_session_scores_trend_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", "description": "Metric name to get trend for", "title": "Metric Name" }, "description": "Metric name to get trend for" }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date From" } }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Date To" } }, { "name": "granularity", "in": "query", "required": false, "schema": { "$ref": "#/components/schemas/AnalyticsGranularity", "default": "day" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreTrendItem" }, "title": "Response Get Session Score Analytics Trend Evaluations Analytics Session Scores Trend Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/session-scores/distribution": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Score Analytics Distribution", "description": "Histogram of session score values for a metric.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_score_analytics_distribution_evaluations_analytics_session_scores_distribution_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", "description": "Session metric name to get distribution for", "title": "Metric Name" }, "description": "Session metric name to get distribution for" }, { "name": "date_from", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date From" }, "description": "ISO 8601 datetime." }, { "name": "date_to", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "description": "ISO 8601 datetime.", "title": "Date To" }, "description": "ISO 8601 datetime." }, { "name": "buckets", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, "description": "Number of histogram buckets (1-100)", "default": 10, "title": "Buckets" }, "description": "Number of histogram buckets (1-100)" } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/ScoreDistributionItem" }, "title": "Response Get Session Score Analytics Distribution Evaluations Analytics Session Scores Distribution Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/session-scores/history/{session_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Score History", "description": "Score evolution for a specific session across re-evaluations over time.\n\nReturns one row per (metric, eval_run) ordered chronologically, showing\nhow the session's scores changed as new traces arrived and the session\nwas re-evaluated. Useful for \"session health over time\" dashboard charts.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_score_history_evaluations_analytics_session_scores_history__session_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "session_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Session Id" } }, { "name": "metric_name", "in": "query", "required": false, "schema": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "description": "Filter to a single metric name.", "title": "Metric Name" }, "description": "Filter to a single metric name." }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "description": "Max data points to return.", "default": 50, "title": "Limit" }, "description": "Max data points to return." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/SessionScoreHistoryItem" }, "title": "Response Get Session Score History Evaluations Analytics Session Scores History Session Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/analytics/session-scores/comparison": { "get": { "tags": [ "evaluations" ], "summary": "Get Session Score Comparison", "description": "Leaderboard: latest score per session for a metric, sorted by value.\n\nShows each session's most recent score for the requested metric,\nranked by value. Use ``sort_order=asc`` to surface the worst-performing\nsessions first (e.g. \"Bottom 10 by agent_reliability\").\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_session_score_comparison_evaluations_analytics_session_scores_comparison_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "metric_name", "in": "query", "required": true, "schema": { "type": "string", "description": "Metric name to rank sessions by.", "title": "Metric Name" }, "description": "Metric name to rank sessions by." }, { "name": "sort_order", "in": "query", "required": false, "schema": { "type": "string", "description": "Sort order: 'asc' (worst first, good for finding problems) or 'desc' (best first).", "default": "asc", "title": "Sort Order" }, "description": "Sort order: 'asc' (worst first, good for finding problems) or 'desc' (best first)." }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 100, "minimum": 1, "description": "Page size.", "default": 20, "title": "Limit" }, "description": "Page size." }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "description": "Number of items to skip.", "default": 0, "title": "Offset" }, "description": "Number of items to skip." } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_SessionComparisonItem_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors": { "post": { "tags": [ "evaluations" ], "summary": "Create Monitor", "description": "Create an evaluation monitor that spawns eval runs on a recurring schedule.\n\nMonitors persist a reusable evaluation configuration (target type, metrics,\nfilters, cadence) and the system automatically creates eval runs at each\nscheduled interval. If ``only_if_changed`` is true (default), runs are\nskipped when no new data has arrived since the previous run.\n\n**Request body fields:**\n\n- **name** *(string, required)*: Human-readable label, e.g. ``\"Daily prod eval\"``.\n- **target_type** *(string, required)*: ``\"TRACE\"`` or ``\"SESSION\"``.\n- **metrics** *(string[], required)*: Metric names to run. Use\n ``GET /evaluations/trace-metrics`` (TRACE) or ``GET /evaluations/session-metrics``\n (SESSION) for available names.\n- **filters** *(object, optional, default {})*: Scope the data the monitor evaluates.\n Pass ``{}`` to match everything. Accepted keys depend on ``target_type``:\n - **TRACE**: ``date_from``, ``date_to`` (ISO 8601), ``status``\n (PENDING/RUNNING/COMPLETED/ERROR), ``session_id``, ``user_id``,\n ``tags`` (string[]), ``name`` (substring match).\n - **SESSION**: ``date_from``, ``date_to``, ``user_id``, ``has_error`` (bool),\n ``tags`` (string[]), ``min_trace_count`` (int).\n- **cadence** *(string, required)*: Firing schedule. Predefined: ``\"every_6h\"``,\n ``\"daily\"``, ``\"weekly\"``. Custom cron: ``\"cron:\"``,\n e.g. ``\"cron:0 3 * * *\"`` (daily 3 AM UTC), ``\"cron:0 6 * * 1-5\"``\n (weekdays 6 AM).\n- **sampling_rate** *(float, optional, default 1.0)*: Fraction of matching\n items to evaluate per run (0.0\u20131.0).\n- **model** *(string, optional)*: LLM model override, e.g. ``\"openai/gpt-4o\"``.\n- **only_if_changed** *(bool, optional, default true)*: Skip the run if no\n new traces/sessions exist since the last run.\n- **signal_weights** *(object, optional, SESSION only)*: Override signal\n aggregation weights. Keys: ``confidence``, ``loop_detection``,\n ``tool_correctness``, ``coherence``.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "create_monitor_evaluations_monitors_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateMonitorRequest" } } } }, "responses": { "201": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MonitorResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "get": { "tags": [ "evaluations" ], "summary": "List Monitors", "description": "List evaluation monitors.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_monitors_evaluations_monitors_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "status", "in": "query", "required": false, "schema": { "anyOf": [ { "$ref": "#/components/schemas/MonitorStatus" }, { "type": "null" } ], "title": "Status" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_MonitorResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors/{monitor_id}": { "get": { "tags": [ "evaluations" ], "summary": "Get Monitor", "description": "Get evaluation monitor detail.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "get_monitor_evaluations_monitors__monitor_id__get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MonitorResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "patch": { "tags": [ "evaluations" ], "summary": "Update Monitor", "description": "Update an evaluation monitor.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "update_monitor_evaluations_monitors__monitor_id__patch", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateMonitorRequest" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MonitorResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } }, "delete": { "tags": [ "evaluations" ], "summary": "Delete Monitor", "description": "Delete an evaluation monitor. Spawned runs are preserved.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "delete_monitor_evaluations_monitors__monitor_id__delete", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "responses": { "204": { "description": "Successful Response" }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors/{monitor_id}/pause": { "post": { "tags": [ "evaluations" ], "summary": "Pause Monitor", "description": "Pause an evaluation monitor. Idempotent if already paused.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "pause_monitor_evaluations_monitors__monitor_id__pause_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MonitorResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors/{monitor_id}/resume": { "post": { "tags": [ "evaluations" ], "summary": "Resume Monitor", "description": "Resume a paused evaluation monitor. Idempotent if already active.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "resume_monitor_evaluations_monitors__monitor_id__resume_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/MonitorResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors/{monitor_id}/trigger": { "post": { "tags": [ "evaluations" ], "summary": "Trigger Monitor", "description": "Force an immediate eval run from a monitor, ignoring cadence.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``\n\nRate limit: ``10/min``", "operationId": "trigger_monitor_evaluations_monitors__monitor_id__trigger_post", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } } ], "responses": { "202": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvalRunResponse" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/evaluations/monitors/{monitor_id}/runs": { "get": { "tags": [ "evaluations" ], "summary": "List Monitor Runs", "description": "List eval runs spawned by a specific monitor.\n\nAuth: ``Bearer`` + ``X-Project-ID`` | ``X-API-Key`` + ``X-Project-Name``", "operationId": "list_monitor_runs_evaluations_monitors__monitor_id__runs_get", "security": [ { "HTTPBearer": [] }, { "ApiKey": [] }, { "ProjectID": [] }, { "ProjectName": [] } ], "parameters": [ { "name": "monitor_id", "in": "path", "required": true, "schema": { "type": "string", "format": "uuid", "title": "Monitor Id" } }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "maximum": 200, "minimum": 1, "default": 50, "title": "Limit" } }, { "name": "offset", "in": "query", "required": false, "schema": { "type": "integer", "minimum": 0, "default": 0, "title": "Offset" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_EvalRunResponse_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } } }, "components": { "schemas": { "APIKeyResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "org_id": { "type": "string", "format": "uuid", "title": "Org Id" }, "key_prefix": { "type": "string", "title": "Key Prefix" }, "name": { "type": "string", "title": "Name" }, "is_active": { "type": "boolean", "title": "Is Active", "description": "False if the key has been revoked or has expired." }, "created_at": { "type": "string", "title": "Created At" }, "expires_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Expires At" }, "raw_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Raw Key", "description": "Shown only at creation time." } }, "type": "object", "required": [ "id", "org_id", "key_prefix", "name", "is_active", "created_at", "expires_at" ], "title": "APIKeyResponse", "description": "Returned after creating an API key \u2014 includes the raw key once." }, "AddMemberRequest": { "properties": { "user_id": { "type": "string", "format": "uuid", "title": "User Id" }, "role": { "$ref": "#/components/schemas/MembershipRole", "default": "MEMBER" } }, "type": "object", "required": [ "user_id" ], "title": "AddMemberRequest", "description": "Payload for inviting a user to an org." }, "AnalyticsBucket": { "properties": { "bucket": { "type": "string", "title": "Bucket" }, "trace_count": { "type": "integer", "title": "Trace Count", "default": 0 }, "error_count": { "type": "integer", "title": "Error Count", "default": 0 }, "avg_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Avg Latency Ms" }, "p50_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "P50 Latency Ms" }, "p90_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "P90 Latency Ms" }, "p99_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "P99 Latency Ms" } }, "type": "object", "required": [ "bucket" ], "title": "AnalyticsBucket", "description": "Time-bucketed trace volume, error, and latency statistics." }, "AnalyticsGranularity": { "type": "string", "enum": [ "hour", "day", "week" ], "title": "AnalyticsGranularity", "description": "Time-bucket granularity for analytics queries." }, "AnalyticsMetric": { "type": "string", "enum": [ "volume", "errors", "latency", "cost", "tokens", "models" ], "title": "AnalyticsMetric", "description": "Available metric types for the analytics endpoint." }, "BatchDeleteRequest": { "properties": { "trace_ids": { "items": { "type": "string", "format": "uuid" }, "type": "array", "maxItems": 500, "minItems": 1, "title": "Trace Ids" } }, "type": "object", "required": [ "trace_ids" ], "title": "BatchDeleteRequest", "description": "Request body for bulk trace deletion.", "example": { "trace_ids": [ "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "b2c3d4e5-f6a7-8901-bcde-f12345678901" ] } }, "BatchDeleteResponse": { "properties": { "deleted": { "type": "integer", "title": "Deleted" } }, "type": "object", "required": [ "deleted" ], "title": "BatchDeleteResponse", "description": "Response for bulk trace deletion (count removed)." }, "BatchTagsRequest": { "properties": { "trace_ids": { "items": { "type": "string", "format": "uuid" }, "type": "array", "maxItems": 500, "minItems": 1, "title": "Trace Ids" }, "add_tags": { "items": { "type": "string" }, "type": "array", "title": "Add Tags" }, "remove_tags": { "items": { "type": "string" }, "type": "array", "title": "Remove Tags" } }, "type": "object", "required": [ "trace_ids" ], "title": "BatchTagsRequest", "description": "Request body for bulk tag manipulation on traces.", "example": { "trace_ids": [ "a1b2c3d4-e5f6-7890-abcd-ef1234567890" ], "add_tags": [ "reviewed", "production" ], "remove_tags": [ "draft" ] } }, "BatchTagsResponse": { "properties": { "updated": { "type": "integer", "title": "Updated" } }, "type": "object", "required": [ "updated" ], "title": "BatchTagsResponse", "description": "Response for bulk tag update (count of traces affected)." }, "CreateAPIKeyRequest": { "properties": { "name": { "type": "string", "maxLength": 255, "minLength": 1, "title": "Name", "description": "Human-readable label for the key" }, "expiration": { "$ref": "#/components/schemas/KeyExpiration", "description": "Key lifetime: 'never' (default, for production) or '90d' (for development)", "default": "never" } }, "type": "object", "required": [ "name" ], "title": "CreateAPIKeyRequest", "description": "Payload for generating a new API key." }, "CreateBatchEvalRunRequest": { "properties": { "trace_ids": { "items": { "type": "string", "format": "uuid" }, "type": "array", "minItems": 1, "title": "Trace Ids", "description": "List of trace UUIDs to evaluate. Duplicates are removed automatically." }, "metrics": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Metrics", "description": "List of metric names to run on each trace. Example: ['task_completion', 'step_efficiency']." }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Optional human-readable label for this run." }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "LLM model string override for the judge. Null uses the system default." } }, "type": "object", "required": [ "trace_ids", "metrics" ], "title": "CreateBatchEvalRunRequest", "description": "Create an eval run for an explicit list of trace IDs.\n\nUse this when the user has manually selected specific traces in the\ndashboard rather than using filter-based selection." }, "CreateBatchSessionEvalRunRequest": { "properties": { "session_ids": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Session Ids", "description": "List of session ID strings." }, "metrics": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Metrics", "description": "Session metric names." }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Human-readable label." }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "LLM model override for judge calls." }, "signal_weights": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Signal Weights", "description": "Override default signal weights." } }, "type": "object", "required": [ "session_ids", "metrics" ], "title": "CreateBatchSessionEvalRunRequest", "description": "Create a session eval run for explicit session IDs." }, "CreateEvalRunRequest": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Optional human-readable label for this run (e.g. 'Weekly prod eval')." }, "metrics": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Metrics", "description": "List of metric names to run. Use GET /evaluations/trace-metrics to see available names. Example: ['task_completion', 'tool_correctness']." }, "filters": { "$ref": "#/components/schemas/EvalRunFilters", "description": "Trace filters to select which traces to evaluate. Omit or leave empty to evaluate all traces in the project." }, "sampling_rate": { "type": "number", "maximum": 1.0, "minimum": 0.0, "title": "Sampling Rate", "description": "Fraction of matching traces to evaluate (0.0 to 1.0). 1.0 = all matching traces, 0.1 = random 10%.", "default": 1.0 }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "LLM model string override for the judge (e.g. 'openai/gpt-4o'). Null uses the system default." } }, "type": "object", "required": [ "metrics" ], "title": "CreateEvalRunRequest", "description": "Create a filtered eval run.\n\nThe system resolves matching traces from the filters, optionally\nsamples a fraction of them, then dispatches a background task to\nrun the requested metrics asynchronously via an LLM judge.\n\n**Typical dashboard flow:**\n1. User selects metrics -> call ``GET /trace-runs/template?metric=task_completion``\n2. Dashboard renders the template as a form with editable filters\n3. User customizes filters/sampling -> frontend builds this request body\n4. Frontend calls ``POST /trace-runs`` with the final body", "example": { "name": "Weekly production eval", "metrics": [ "task_completion", "tool_correctness", "step_efficiency" ], "filters": { "date_from": "2026-03-22T00:00:00Z", "date_to": "2026-03-29T00:00:00Z", "status": "COMPLETED" }, "sampling_rate": 0.5, "model": "gpt-5.4" } }, "CreateMonitorRequest": { "properties": { "name": { "type": "string", "title": "Name", "description": "Human-readable name for the monitor, e.g. 'Daily prod trace eval'." }, "target_type": { "type": "string", "title": "Target Type", "description": "Evaluation scope: ``'TRACE'`` for trace-level metrics or ``'SESSION'`` for session-level metrics." }, "metrics": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Metrics", "description": "Metric names to run on each scheduled eval. For TRACE monitors use ``GET /evaluations/trace-metrics``; for SESSION monitors use ``GET /evaluations/session-metrics`` to list available names. Example: ``['task_completion', 'step_efficiency']``." }, "filters": { "additionalProperties": true, "type": "object", "title": "Filters", "description": "JSON object defining which traces/sessions the monitor targets. Pass ``{}`` to match everything in the project. **TRACE monitors** accept: ``date_from`` (ISO 8601), ``date_to`` (ISO 8601), ``status`` (PENDING|RUNNING|COMPLETED|ERROR), ``session_id``, ``user_id``, ``tags`` (string array, ANY match), ``name`` (substring, case-insensitive). **SESSION monitors** accept: ``date_from``, ``date_to``, ``user_id``, ``has_error`` (bool), ``tags``, ``min_trace_count`` (int). Example: ``{\"status\": \"COMPLETED\", \"tags\": [\"production\"]}``." }, "sampling_rate": { "type": "number", "maximum": 1.0, "minimum": 0.0, "title": "Sampling Rate", "description": "Fraction of matching traces/sessions to evaluate per run. 1.0 = all, 0.1 = random 10%.", "default": 1.0 }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "LLM model override for judge calls (e.g. ``'openai/gpt-4o'``). Uses system default if null." }, "cadence": { "type": "string", "title": "Cadence", "description": "How often the monitor fires. Predefined intervals: ``'every_6h'``, ``'daily'``, ``'weekly'``. Custom cron: ``'cron:<5-part expression>'`` where the five parts are ``minute hour day-of-month month day-of-week``. Examples: ``'cron:0 3 * * *'`` (daily at 3 AM UTC), ``'cron:0 6 * * 1-5'`` (weekdays at 6 AM), ``'cron:0 */4 * * *'`` (every 4 hours)." }, "only_if_changed": { "type": "boolean", "title": "Only If Changed", "description": "When true, the scheduled run is skipped if no new traces/sessions have arrived since the last run, saving LLM costs. Set to false to always run on schedule regardless of new data.", "default": true }, "signal_weights": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Signal Weights", "description": "Override default signal weights for session-level aggregation. **Only valid for SESSION monitors** (rejected for TRACE). Keys: ``confidence``, ``loop_detection``, ``tool_correctness``, ``coherence``. Defaults: confidence=1.0, loop_detection=1.0, tool_correctness=0.8, coherence=1.0. Example: ``{\"confidence\": 1.0, \"loop_detection\": 1.5}``." } }, "type": "object", "required": [ "name", "target_type", "metrics", "cadence" ], "title": "CreateMonitorRequest", "description": "Create an evaluation monitor that spawns runs on a cadence.", "example": { "name": "Daily production trace eval", "target_type": "TRACE", "metrics": [ "task_completion", "tool_correctness" ], "filters": { "status": "COMPLETED" }, "sampling_rate": 0.3, "cadence": "daily", "only_if_changed": true, "model": "gpt-5.4" } }, "CreateOrganizationRequest": { "properties": { "name": { "type": "string", "maxLength": 255, "minLength": 1, "title": "Name" } }, "type": "object", "required": [ "name" ], "title": "CreateOrganizationRequest", "description": "Payload for creating a new organization." }, "CreateProjectRequest": { "properties": { "name": { "type": "string", "maxLength": 255, "minLength": 1, "title": "Name" }, "description": { "type": "string", "maxLength": 2000, "title": "Description", "default": "" } }, "type": "object", "required": [ "name" ], "title": "CreateProjectRequest", "description": "Payload for creating a new project." }, "CreateSessionEvalRunRequest": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Human-readable label." }, "metrics": { "items": { "type": "string" }, "type": "array", "minItems": 1, "title": "Metrics", "description": "Session metric names (e.g. ['agent_reliability'])." }, "filters": { "$ref": "#/components/schemas/SessionEvalRunFilters" }, "sampling_rate": { "type": "number", "maximum": 1.0, "minimum": 0.0, "title": "Sampling Rate", "description": "Fraction of sessions to evaluate.", "default": 1.0 }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "LLM model override for judge calls." }, "signal_weights": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Signal Weights", "description": "Override default signal weights." } }, "type": "object", "required": [ "metrics" ], "title": "CreateSessionEvalRunRequest", "description": "Create a filter-based session eval run.", "example": { "name": "Agent reliability check", "metrics": [ "agent_reliability", "agent_consistency" ], "filters": { "date_from": "2026-03-22T00:00:00Z", "date_to": "2026-03-29T00:00:00Z" }, "model": "gpt-5.4" } }, "CreateTraceScoreRequest": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id", "description": "Trace UUID to attach the score to." }, "name": { "type": "string", "title": "Name", "description": "Metric/score name (e.g. 'task_completion', 'quality', 'thumbs_up')." }, "value": { "type": "string", "title": "Value", "description": "Score value as string. For NUMERIC: '0.85', for BOOLEAN: 'true', for CATEGORICAL: 'PASS'." }, "data_type": { "$ref": "#/components/schemas/ScoreDataType", "description": "Value type: NUMERIC, BOOLEAN, CATEGORICAL.", "default": "NUMERIC" }, "source": { "$ref": "#/components/schemas/ScoreSource", "description": "Score origin: ANNOTATION (human) or PROGRAMMATIC (SDK).", "default": "ANNOTATION" }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason", "description": "Optional explanation or annotation note." }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata", "description": "Optional metadata object." } }, "type": "object", "required": [ "trace_id", "name", "value" ], "title": "CreateTraceScoreRequest", "description": "Manually create a trace score (annotation or programmatic).\n\nUsed by the dashboard for human annotations or by the SDK for\nprogrammatic score submission.", "example": { "trace_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "name": "accuracy", "data_type": "NUMERIC", "value": "0.92", "source": "HUMAN", "comment": "Verified correct response" } }, "EvalRunFilters": { "properties": { "date_from": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Date From", "description": "ISO 8601 datetime string (e.g. '2025-01-15T00:00:00Z'). Include traces started on or after this time." }, "date_to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Date To", "description": "ISO 8601 datetime string (e.g. '2025-02-01T00:00:00Z'). Include traces started before this time (exclusive)." }, "status": { "anyOf": [ { "$ref": "#/components/schemas/TraceStatus" }, { "type": "null" } ], "description": "Trace status: PENDING, RUNNING, COMPLETED, or ERROR." }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id", "description": "Exact session ID string. Only traces belonging to this session." }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id", "description": "Exact user ID string. Only traces from this user." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "List of tag strings (e.g. ['production', 'v2']). Traces matching ANY of these tags are included." }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Substring match on trace name (case-insensitive). E.g. 'booking' matches 'Flight Booking Agent'." } }, "type": "object", "title": "EvalRunFilters", "description": "Trace filters for a filtered eval run.\n\nThese mirror the GET /traces query parameters so the frontend can\nreuse the same filter UI components." }, "EvalRunResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "status": { "$ref": "#/components/schemas/EvaluationStatus" }, "metric_names": { "items": { "type": "string" }, "type": "array", "title": "Metric Names" }, "total_traces": { "type": "integer", "title": "Total Traces" }, "evaluated_count": { "type": "integer", "title": "Evaluated Count" }, "failed_count": { "type": "integer", "title": "Failed Count" }, "created_at": { "type": "string", "title": "Created At" }, "completed_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Completed At" }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id" }, "target_type": { "type": "string", "title": "Target Type" }, "filters": { "additionalProperties": true, "type": "object", "title": "Filters" }, "sampling_rate": { "type": "number", "title": "Sampling Rate" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" }, "monitor_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Monitor Id" }, "error_message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error Message" } }, "type": "object", "required": [ "id", "name", "status", "metric_names", "total_traces", "evaluated_count", "failed_count", "created_at", "completed_at", "project_id", "target_type", "filters", "sampling_rate", "model", "monitor_id", "error_message" ], "title": "EvalRunResponse", "description": "Full eval run representation used by both list and detail endpoints." }, "EvalRunTemplate": { "properties": { "metric": { "$ref": "#/components/schemas/MetricInfo" }, "filters": { "$ref": "#/components/schemas/EvalRunFilters" }, "sampling_rate": { "type": "number", "title": "Sampling Rate" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" } }, "type": "object", "required": [ "metric", "filters", "sampling_rate", "model" ], "title": "EvalRunTemplate", "description": "Pre-filled template the dashboard renders as an editable form." }, "EvaluationStatus": { "type": "string", "enum": [ "PENDING", "RUNNING", "COMPLETED", "FAILED" ], "title": "EvaluationStatus", "description": "Lifecycle status of an evaluation job." }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "KeyExpiration": { "type": "string", "enum": [ "never", "90d" ], "title": "KeyExpiration", "description": "Supported API key lifetime options." }, "MembershipResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "user_id": { "type": "string", "format": "uuid", "title": "User Id" }, "org_id": { "type": "string", "format": "uuid", "title": "Org Id" }, "role": { "$ref": "#/components/schemas/MembershipRole" }, "display_name": { "type": "string", "title": "Display Name" }, "email": { "type": "string", "title": "Email" }, "created_at": { "type": "string", "title": "Created At" } }, "type": "object", "required": [ "id", "user_id", "org_id", "role", "display_name", "email", "created_at" ], "title": "MembershipResponse", "description": "Membership entry with user display info." }, "MembershipRole": { "type": "string", "enum": [ "OWNER", "ADMIN", "MEMBER" ], "title": "MembershipRole", "description": "Role a user holds within an organization." }, "MetricInfo": { "properties": { "name": { "type": "string", "title": "Name" }, "description": { "type": "string", "title": "Description" }, "category": { "type": "string", "title": "Category" }, "default_threshold": { "type": "number", "title": "Default Threshold" }, "prompt_preview": { "items": { "$ref": "#/components/schemas/PromptPreview" }, "type": "array", "title": "Prompt Preview" } }, "type": "object", "required": [ "name", "description", "category", "default_threshold", "prompt_preview" ], "title": "MetricInfo", "description": "Full metric info including threshold and prompt previews." }, "MetricSummary": { "properties": { "name": { "type": "string", "title": "Name" }, "description": { "type": "string", "title": "Description" }, "category": { "type": "string", "title": "Category" } }, "type": "object", "required": [ "name", "description", "category" ], "title": "MetricSummary", "description": "Lightweight metric info for the list endpoint." }, "MonitorResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id" }, "name": { "type": "string", "title": "Name" }, "target_type": { "type": "string", "title": "Target Type" }, "metric_names": { "items": { "type": "string" }, "type": "array", "title": "Metric Names" }, "filters": { "additionalProperties": true, "type": "object", "title": "Filters" }, "sampling_rate": { "type": "number", "title": "Sampling Rate" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" }, "cadence": { "type": "string", "title": "Cadence" }, "only_if_changed": { "type": "boolean", "title": "Only If Changed" }, "status": { "type": "string", "title": "Status" }, "last_run_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Last Run At" }, "last_run_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Last Run Id" }, "next_run_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Next Run At" }, "created_at": { "type": "string", "title": "Created At" }, "updated_at": { "type": "string", "title": "Updated At" } }, "type": "object", "required": [ "id", "project_id", "name", "target_type", "metric_names", "filters", "sampling_rate", "model", "cadence", "only_if_changed", "status", "last_run_at", "last_run_id", "next_run_at", "created_at", "updated_at" ], "title": "MonitorResponse", "description": "Full evaluation monitor representation." }, "MonitorStatus": { "type": "string", "enum": [ "ACTIVE", "PAUSED" ], "title": "MonitorStatus", "description": "Lifecycle status of an evaluation monitor." }, "MyOrganizationResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "name": { "type": "string", "title": "Name" }, "created_at": { "type": "string", "title": "Created At" }, "role": { "$ref": "#/components/schemas/MembershipRole" } }, "type": "object", "required": [ "id", "name", "created_at", "role" ], "title": "MyOrganizationResponse", "description": "Organization with the caller's role included." }, "OrganizationResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "name": { "type": "string", "title": "Name" }, "created_at": { "type": "string", "title": "Created At" } }, "type": "object", "required": [ "id", "name", "created_at" ], "title": "OrganizationResponse", "description": "Public representation of an organization." }, "PaginatedResponse_EvalRunResponse_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/EvalRunResponse" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[EvalRunResponse]" }, "PaginatedResponse_MonitorResponse_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/MonitorResponse" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[MonitorResponse]" }, "PaginatedResponse_SessionComparisonItem_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/SessionComparisonItem" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[SessionComparisonItem]" }, "PaginatedResponse_SessionScoreResponse_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/SessionScoreResponse" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[SessionScoreResponse]" }, "PaginatedResponse_SessionSummary_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/SessionSummary" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[SessionSummary]" }, "PaginatedResponse_TraceListItem_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/TraceListItem" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[TraceListItem]" }, "PaginatedResponse_TraceScoreResponse_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/TraceScoreResponse" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[TraceScoreResponse]" }, "PaginatedResponse_UserSummary_": { "properties": { "items": { "items": { "$ref": "#/components/schemas/UserSummary" }, "type": "array", "title": "Items" }, "total": { "type": "integer", "title": "Total" }, "limit": { "type": "integer", "title": "Limit" }, "offset": { "type": "integer", "title": "Offset" } }, "type": "object", "required": [ "items", "total", "limit", "offset" ], "title": "PaginatedResponse[UserSummary]" }, "ProjectResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "org_id": { "type": "string", "format": "uuid", "title": "Org Id" }, "name": { "type": "string", "title": "Name" }, "description": { "type": "string", "title": "Description" }, "created_at": { "type": "string", "title": "Created At" } }, "type": "object", "required": [ "id", "org_id", "name", "description", "created_at" ], "title": "ProjectResponse", "description": "Public representation of a project." }, "PromptPreview": { "properties": { "stage": { "type": "string", "title": "Stage" }, "prompt": { "type": "string", "title": "Prompt" } }, "type": "object", "required": [ "stage", "prompt" ], "title": "PromptPreview", "description": "Actual LLM prompt text for one evaluation stage, rendered with sample data." }, "ProviderInfo": { "properties": { "key": { "type": "string", "title": "Key" }, "name": { "type": "string", "title": "Name" }, "description": { "type": "string", "title": "Description" }, "available": { "type": "boolean", "title": "Available" }, "message": { "type": "string", "title": "Message" } }, "type": "object", "required": [ "key", "name", "description", "available", "message" ], "title": "ProviderInfo", "description": "Availability info for a single LLM provider." }, "ScoreDataType": { "type": "string", "enum": [ "NUMERIC", "BOOLEAN", "CATEGORICAL" ], "title": "ScoreDataType", "description": "Data type for a trace score value." }, "ScoreDistributionItem": { "properties": { "bucket": { "type": "integer", "title": "Bucket" }, "bucket_min": { "type": "number", "title": "Bucket Min" }, "bucket_max": { "type": "number", "title": "Bucket Max" }, "count": { "type": "integer", "title": "Count" } }, "type": "object", "required": [ "bucket", "bucket_min", "bucket_max", "count" ], "title": "ScoreDistributionItem", "description": "Histogram bucket for score distribution." }, "ScoreSource": { "type": "string", "enum": [ "AUTOMATED", "ANNOTATION", "PROGRAMMATIC" ], "title": "ScoreSource", "description": "Who produced the score judgment (not how it arrived).\n\nAUTOMATED -- the eval system's LLM judge produced this score\nANNOTATION -- a human assigned this score manually\nPROGRAMMATIC -- external code submitted this score via API/SDK" }, "ScoreStatus": { "type": "string", "enum": [ "SUCCESS", "FAILED", "PENDING" ], "title": "ScoreStatus", "description": "Outcome of a trace score evaluation attempt." }, "ScoreSummaryItem": { "properties": { "metric_name": { "type": "string", "title": "Metric Name" }, "avg_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Avg Score" }, "min_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Min Score" }, "max_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Max Score" }, "median_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Median Score" }, "success_count": { "type": "integer", "title": "Success Count" }, "failed_count": { "type": "integer", "title": "Failed Count" }, "latest_score_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Latest Score At" } }, "type": "object", "required": [ "metric_name", "avg_score", "min_score", "max_score", "median_score", "success_count", "failed_count", "latest_score_at" ], "title": "ScoreSummaryItem", "description": "Aggregated score summary for one metric." }, "ScoreTrendItem": { "properties": { "bucket": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Bucket" }, "metric_name": { "type": "string", "title": "Metric Name" }, "avg_score": { "type": "number", "title": "Avg Score" }, "count": { "type": "integer", "title": "Count" } }, "type": "object", "required": [ "bucket", "metric_name", "avg_score", "count" ], "title": "ScoreTrendItem", "description": "Time-bucketed average score for a metric." }, "SessionAnalyticsBucket": { "properties": { "bucket": { "type": "string", "title": "Bucket" }, "session_count": { "type": "integer", "title": "Session Count", "default": 0 }, "avg_traces_per_session": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Avg Traces Per Session" }, "avg_session_duration_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Avg Session Duration Ms" } }, "type": "object", "required": [ "bucket" ], "title": "SessionAnalyticsBucket", "description": "Time-bucketed session statistics." }, "SessionComparisonItem": { "properties": { "session_id": { "type": "string", "title": "Session Id" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score" }, "evaluated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Evaluated At" }, "eval_run_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Eval Run Id" } }, "type": "object", "required": [ "session_id", "score", "evaluated_at", "eval_run_id" ], "title": "SessionComparisonItem", "description": "A session's latest score for a metric, used in leaderboard/ranking views." }, "SessionDeleteResponse": { "properties": { "deleted": { "type": "integer", "title": "Deleted" } }, "type": "object", "required": [ "deleted" ], "title": "SessionDeleteResponse", "description": "Response for session deletion (count of traces removed)." }, "SessionDetail": { "properties": { "session_id": { "type": "string", "title": "Session Id" }, "trace_count": { "type": "integer", "title": "Trace Count" }, "first_trace_at": { "type": "string", "title": "First Trace At" }, "last_trace_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Last Trace At" }, "total_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Total Latency Ms" }, "has_error": { "type": "boolean", "title": "Has Error" }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "total_span_count": { "type": "integer", "title": "Total Span Count", "default": 0 }, "total_tokens": { "type": "integer", "title": "Total Tokens", "default": 0 }, "total_cost": { "type": "number", "title": "Total Cost", "default": 0.0 }, "traces": { "items": { "$ref": "#/components/schemas/TraceResponse" }, "type": "array", "title": "Traces" } }, "type": "object", "required": [ "session_id", "trace_count", "first_trace_at", "last_trace_at", "total_latency_ms", "has_error", "user_id", "tags" ], "title": "SessionDetail", "description": "Single session with its ordered traces (full detail)." }, "SessionEvalRunFilters": { "properties": { "date_from": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Date From", "description": "ISO 8601 datetime." }, "date_to": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Date To", "description": "ISO 8601 datetime." }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id", "description": "Exact user ID string." }, "has_error": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Has Error", "description": "Only sessions with/without errors." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "Traces matching ANY of these tags." }, "min_trace_count": { "anyOf": [ { "type": "integer", "minimum": 1.0 }, { "type": "null" } ], "title": "Min Trace Count", "description": "Minimum traces in a session." } }, "type": "object", "title": "SessionEvalRunFilters", "description": "Filters for session-level evaluation runs." }, "SessionScoreHistoryItem": { "properties": { "metric_name": { "type": "string", "title": "Metric Name" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score" }, "eval_run_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Eval Run Id" }, "created_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Created At" }, "status": { "type": "string", "title": "Status" } }, "type": "object", "required": [ "metric_name", "score", "eval_run_id", "created_at", "status" ], "title": "SessionScoreHistoryItem", "description": "A single data point in a session's score evolution over time." }, "SessionScoreResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "session_id": { "type": "string", "title": "Session Id" }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id" }, "name": { "type": "string", "title": "Name" }, "data_type": { "type": "string", "title": "Data Type" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value" }, "source": { "type": "string", "title": "Source" }, "status": { "type": "string", "title": "Status" }, "eval_run_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Eval Run Id" }, "author_user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Author User Id" }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "created_at": { "type": "string", "title": "Created At" }, "updated_at": { "type": "string", "title": "Updated At" } }, "type": "object", "required": [ "id", "session_id", "project_id", "name", "data_type", "value", "source", "status", "eval_run_id", "author_user_id", "reason", "metadata", "created_at", "updated_at" ], "title": "SessionScoreResponse", "description": "Full session score representation." }, "SessionSortBy": { "type": "string", "enum": [ "recent", "trace_count", "latency", "cost" ], "title": "SessionSortBy", "description": "Columns available for sorting session list results." }, "SessionSummary": { "properties": { "session_id": { "type": "string", "title": "Session Id" }, "trace_count": { "type": "integer", "title": "Trace Count" }, "first_trace_at": { "type": "string", "title": "First Trace At" }, "last_trace_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Last Trace At" }, "total_latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Total Latency Ms" }, "has_error": { "type": "boolean", "title": "Has Error" }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "total_span_count": { "type": "integer", "title": "Total Span Count", "default": 0 }, "total_tokens": { "type": "integer", "title": "Total Tokens", "default": 0 }, "total_cost": { "type": "number", "title": "Total Cost", "default": 0.0 } }, "type": "object", "required": [ "session_id", "trace_count", "first_trace_at", "last_trace_at", "total_latency_ms", "has_error", "user_id", "tags" ], "title": "SessionSummary", "description": "Aggregated summary for a session." }, "SortOrder": { "type": "string", "enum": [ "asc", "desc" ], "title": "SortOrder", "description": "Generic ascending / descending sort direction." }, "SpanCreate": { "properties": { "span_id": { "type": "string", "format": "uuid", "title": "Span Id" }, "parent_span_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Parent Span Id" }, "name": { "type": "string", "maxLength": 512, "minLength": 1, "title": "Name" }, "kind": { "$ref": "#/components/schemas/SpanKind", "default": "OTHER" }, "status": { "$ref": "#/components/schemas/SpanStatusCode", "default": "UNSET" }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "model": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Model" }, "token_usage": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Token Usage" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "started_at": { "type": "string", "format": "date-time", "title": "Started At" }, "ended_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Ended At" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" }, "completion_start_time": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Completion Start Time" }, "model_parameters": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Model Parameters" }, "cost": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Cost" } }, "type": "object", "required": [ "name", "started_at" ], "title": "SpanCreate", "description": "Schema for a span inside a trace-ingestion request.", "example": { "span_id": "f1e2d3c4-b5a6-7890-abcd-ef1234567890", "name": "tool-call-search", "kind": "TOOL", "status": "OK", "input": { "query": "password reset" }, "output": { "results": [ { "title": "Reset your password", "url": "/help/reset" } ] }, "started_at": "2026-03-29T10:00:00.500Z", "ended_at": "2026-03-29T10:00:01.200Z" } }, "SpanKind": { "type": "string", "enum": [ "AGENT", "TOOL", "LLM", "RETRIEVER", "CHAIN", "EMBEDDING", "OTHER" ], "title": "SpanKind", "description": "Categorises what a span represents in an agentic workflow." }, "SpanResponse": { "properties": { "span_id": { "type": "string", "format": "uuid", "title": "Span Id" }, "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "parent_span_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Parent Span Id" }, "name": { "type": "string", "title": "Name" }, "kind": { "$ref": "#/components/schemas/SpanKind" }, "status": { "$ref": "#/components/schemas/SpanStatusCode" }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" }, "token_usage": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Token Usage" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "started_at": { "type": "string", "title": "Started At" }, "ended_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ended At" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" }, "completion_start_time": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Completion Start Time" }, "model_parameters": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Model Parameters" }, "cost": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Cost" }, "latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency Ms" }, "time_to_first_token_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Time To First Token Ms" } }, "type": "object", "required": [ "span_id", "trace_id", "parent_span_id", "name", "kind", "status", "input", "output", "model", "token_usage", "metadata", "started_at", "ended_at", "error", "completion_start_time", "model_parameters", "cost" ], "title": "SpanResponse", "description": "Public span representation." }, "SpanStatusCode": { "type": "string", "enum": [ "UNSET", "OK", "ERROR" ], "title": "SpanStatusCode", "description": "Mirrors the OTel StatusCode for a span's outcome." }, "SpanUpdate": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 512 }, { "type": "null" } ], "title": "Name" }, "kind": { "anyOf": [ { "$ref": "#/components/schemas/SpanKind" }, { "type": "null" } ] }, "status": { "anyOf": [ { "$ref": "#/components/schemas/SpanStatusCode" }, { "type": "null" } ] }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "model": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Model" }, "token_usage": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Token Usage" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "ended_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Ended At" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" }, "completion_start_time": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Completion Start Time" }, "model_parameters": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Model Parameters" }, "cost": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Cost" } }, "type": "object", "title": "SpanUpdate", "description": "Partial-update schema for a span (all fields optional)." }, "SpansAccepted": { "properties": { "span_ids": { "items": { "type": "string", "format": "uuid" }, "type": "array", "title": "Span Ids" } }, "type": "object", "required": [ "span_ids" ], "title": "SpansAccepted", "description": "Response body for successfully added spans." }, "TokenCostBucket": { "properties": { "bucket": { "type": "string", "title": "Bucket" }, "total_tokens": { "type": "integer", "title": "Total Tokens", "default": 0 }, "prompt_tokens": { "type": "integer", "title": "Prompt Tokens", "default": 0 }, "completion_tokens": { "type": "integer", "title": "Completion Tokens", "default": 0 }, "total_cost": { "type": "number", "title": "Total Cost", "default": 0.0 } }, "type": "object", "required": [ "bucket" ], "title": "TokenCostBucket", "description": "Time-bucketed token usage and cost aggregation." }, "TopModel": { "properties": { "model": { "type": "string", "title": "Model" }, "call_count": { "type": "integer", "title": "Call Count" }, "total_tokens": { "type": "integer", "title": "Total Tokens", "default": 0 }, "total_cost": { "type": "number", "title": "Total Cost", "default": 0.0 } }, "type": "object", "required": [ "model", "call_count" ], "title": "TopModel", "description": "LLM model ranked by call count within a time window." }, "TraceAccepted": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "task_id": { "type": "string", "title": "Task Id" } }, "type": "object", "required": [ "trace_id", "task_id" ], "title": "TraceAccepted", "description": "Response body for a successfully enqueued trace.", "example": { "trace_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "task_id": "celery-task-xyz-789" } }, "TraceCreate": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "name": { "type": "string", "maxLength": 512, "minLength": 1, "title": "Name" }, "status": { "$ref": "#/components/schemas/TraceStatus", "default": "COMPLETED" }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "started_at": { "type": "string", "format": "date-time", "title": "Started At" }, "ended_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Ended At" }, "session_id": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Session Id" }, "user_id": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "User Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "environment": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Environment" }, "release": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Release" }, "spans": { "items": { "$ref": "#/components/schemas/SpanCreate" }, "type": "array", "maxItems": 500, "title": "Spans" } }, "type": "object", "required": [ "name", "started_at" ], "title": "TraceCreate", "description": "Schema for the ``POST /traces`` request body.", "example": { "trace_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", "name": "customer-support-agent", "status": "COMPLETED", "input": { "query": "How do I reset my password?" }, "output": { "response": "You can reset your password from Settings > Security." }, "metadata": { "environment": "production", "sdk_version": "0.5.0" }, "started_at": "2026-03-29T10:00:00Z", "ended_at": "2026-03-29T10:00:02.450Z", "session_id": "sess-abc-123", "user_id": "user-42", "tags": [ "production", "support" ], "spans": [ { "span_id": "f1e2d3c4-b5a6-7890-abcd-ef1234567890", "name": "llm-call", "kind": "LLM", "status": "OK", "input": { "messages": [ { "role": "user", "content": "How do I reset my password?" } ] }, "output": { "choices": [ { "message": { "content": "You can reset your password from Settings > Security." } } ] }, "started_at": "2026-03-29T10:00:00.100Z", "ended_at": "2026-03-29T10:00:02.400Z", "model": "gpt-5.4", "token_usage": { "prompt_tokens": 45, "completion_tokens": 22, "total_tokens": 67 } } ] } }, "TraceListItem": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "name": { "type": "string", "title": "Name" }, "status": { "$ref": "#/components/schemas/TraceStatus" }, "started_at": { "type": "string", "title": "Started At" }, "ended_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ended At" }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id" }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "environment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Environment" }, "release": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Release" }, "latency_ms": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency Ms" }, "span_count": { "type": "integer", "title": "Span Count", "default": 0 }, "total_tokens": { "type": "integer", "title": "Total Tokens", "default": 0 }, "total_cost": { "type": "number", "title": "Total Cost", "default": 0.0 } }, "type": "object", "required": [ "trace_id", "name", "status", "started_at", "ended_at", "session_id", "user_id", "tags" ], "title": "TraceListItem", "description": "Lightweight trace summary used in list responses." }, "TraceResponse": { "properties": { "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id" }, "name": { "type": "string", "title": "Name" }, "status": { "$ref": "#/components/schemas/TraceStatus" }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "started_at": { "type": "string", "title": "Started At" }, "ended_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Ended At" }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id" }, "user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "environment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Environment" }, "release": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Release" }, "spans": { "items": { "$ref": "#/components/schemas/SpanResponse" }, "type": "array", "title": "Spans" } }, "type": "object", "required": [ "trace_id", "project_id", "name", "status", "input", "output", "metadata", "started_at", "ended_at", "session_id", "user_id", "tags" ], "title": "TraceResponse", "description": "Public trace representation (single-item detail)." }, "TraceScoreResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "trace_id": { "type": "string", "format": "uuid", "title": "Trace Id" }, "name": { "type": "string", "title": "Name" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value" }, "status": { "$ref": "#/components/schemas/ScoreStatus" }, "source": { "$ref": "#/components/schemas/ScoreSource" }, "created_at": { "type": "string", "title": "Created At" }, "project_id": { "type": "string", "format": "uuid", "title": "Project Id" }, "data_type": { "$ref": "#/components/schemas/ScoreDataType" }, "eval_run_id": { "anyOf": [ { "type": "string", "format": "uuid" }, { "type": "null" } ], "title": "Eval Run Id" }, "author_user_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Author User Id" }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason" }, "environment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Environment" }, "config_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Config Id" }, "metadata": { "additionalProperties": true, "type": "object", "title": "Metadata" }, "updated_at": { "type": "string", "title": "Updated At" } }, "type": "object", "required": [ "id", "trace_id", "name", "value", "status", "source", "created_at", "project_id", "data_type", "eval_run_id", "author_user_id", "reason", "environment", "config_id", "metadata", "updated_at" ], "title": "TraceScoreResponse", "description": "Full trace score representation used by both list and detail endpoints." }, "TraceSortBy": { "type": "string", "enum": [ "started_at", "ended_at", "name", "latency", "status" ], "title": "TraceSortBy", "description": "Columns available for sorting trace list results." }, "TraceStatus": { "type": "string", "enum": [ "PENDING", "RUNNING", "COMPLETED", "ERROR" ], "title": "TraceStatus", "description": "High-level status of a trace." }, "TraceUpdate": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 512 }, { "type": "null" } ], "title": "Name" }, "status": { "anyOf": [ { "$ref": "#/components/schemas/TraceStatus" }, { "type": "null" } ] }, "input": { "anyOf": [ {}, { "type": "null" } ], "title": "Input" }, "output": { "anyOf": [ {}, { "type": "null" } ], "title": "Output" }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata" }, "ended_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Ended At" }, "session_id": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Session Id" }, "user_id": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "User Id" }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags" }, "environment": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Environment" }, "release": { "anyOf": [ { "type": "string", "maxLength": 255 }, { "type": "null" } ], "title": "Release" } }, "type": "object", "title": "TraceUpdate", "description": "Partial-update schema for a trace (all fields optional)." }, "UpdateMemberRoleRequest": { "properties": { "role": { "$ref": "#/components/schemas/MembershipRole" } }, "type": "object", "required": [ "role" ], "title": "UpdateMemberRoleRequest", "description": "Payload for changing a member's role." }, "UpdateMonitorRequest": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "metrics": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Metrics" }, "filters": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Filters" }, "sampling_rate": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Sampling Rate" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" }, "cadence": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Cadence" }, "only_if_changed": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Only If Changed" }, "signal_weights": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Signal Weights" } }, "type": "object", "title": "UpdateMonitorRequest", "description": "Partial update of a monitor's configuration." }, "UpdateOrganizationRequest": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 255, "minLength": 1 }, { "type": "null" } ], "title": "Name" } }, "type": "object", "title": "UpdateOrganizationRequest", "description": "Payload for updating an organization. Only provided fields are changed." }, "UpdateProjectRequest": { "properties": { "name": { "anyOf": [ { "type": "string", "maxLength": 255, "minLength": 1 }, { "type": "null" } ], "title": "Name" }, "description": { "anyOf": [ { "type": "string", "maxLength": 2000 }, { "type": "null" } ], "title": "Description" } }, "type": "object", "title": "UpdateProjectRequest", "description": "Payload for updating a project. Only provided fields are changed." }, "UpdateTraceScoreRequest": { "properties": { "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value", "description": "New score value (e.g. '0.9' for NUMERIC, 'true' for BOOLEAN)." }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason", "description": "Updated reason or annotation note." }, "metadata": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Updated metadata object (replaces existing)." } }, "type": "object", "title": "UpdateTraceScoreRequest", "description": "Editable fields for a trace score.\n\nAll fields are optional -- only provided fields are updated.\n``status`` is set to SUCCESS automatically on save, and\n``source`` is changed to ANNOTATION to indicate human edit.\n``updated_at`` is also set automatically." }, "UserProfileResponse": { "properties": { "id": { "type": "string", "format": "uuid", "title": "Id" }, "email": { "type": "string", "title": "Email" }, "display_name": { "type": "string", "title": "Display Name" }, "created_at": { "type": "string", "title": "Created At" }, "last_sign_in_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Last Sign In At" } }, "type": "object", "required": [ "id", "email", "display_name", "created_at", "last_sign_in_at" ], "title": "UserProfileResponse", "description": "Current authenticated user profile." }, "UserSummary": { "properties": { "user_id": { "type": "string", "title": "User Id" }, "trace_count": { "type": "integer", "title": "Trace Count" }, "first_seen": { "type": "string", "title": "First Seen" }, "last_seen": { "type": "string", "title": "Last Seen" }, "error_count": { "type": "integer", "title": "Error Count" } }, "type": "object", "required": [ "user_id", "trace_count", "first_seen", "last_seen", "error_count" ], "title": "UserSummary", "description": "Aggregated trace statistics for a single end-user." }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "msg": { "type": "string", "title": "Message" }, "type": { "type": "string", "title": "Error Type" }, "input": { "title": "Input" }, "ctx": { "type": "object", "title": "Context" } }, "type": "object", "required": [ "loc", "msg", "type" ], "title": "ValidationError" } }, "securitySchemes": { "HTTPBearer": { "type": "http", "scheme": "bearer" }, "ApiKey": { "type": "apiKey", "in": "header", "name": "X-API-Key" }, "ProjectID": { "type": "apiKey", "in": "header", "name": "X-Project-ID" }, "ProjectName": { "type": "apiKey", "in": "header", "name": "X-Project-Name" } } }, "servers": [ { "url": "https://api.pandaprobe.com", "description": "Production" } ], "tags": [ { "name": "health", "description": "Service health and readiness probes." }, { "name": "user", "description": "Authenticated user profile." }, { "name": "traces", "description": "Trace ingestion, querying, updating, and analytics." }, { "name": "sessions", "description": "Session-based trace aggregation and analytics." }, { "name": "evaluations", "description": "Evaluation runs, scores, analytics, and monitoring." }, { "name": "organizations", "description": "Organization management and membership." }, { "name": "projects", "description": "Project CRUD within organizations." }, { "name": "api-keys", "description": "API key lifecycle management." } ] }