Langfuse Evaluations API
CRUD operations for evals: define evaluators (LLM-as-judge, code, human), trigger evaluations on traces or dataset runs, and read back scores and reasoning.
OpenAPI Specification
openapi: 3.0.1
info:
title: langfuse
version: ''
description: >-
## Authentication
Authenticate with the API using [Basic
Auth](https://en.wikipedia.org/wiki/Basic_access_authentication), get API
keys in the project settings:
- username: Langfuse Public Key
- password: Langfuse Secret Key
## Exports
- OpenAPI spec: https://cloud.langfuse.com/generated/api/openapi.yml
paths:
/api/public/annotation-queues:
get:
description: Get all annotation queues
operationId: annotationQueues_listQueues
tags:
- AnnotationQueues
parameters:
- name: page
in: query
description: page number, starts at 1
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
description: limit of items per page
required: false
schema:
type: integer
nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/PaginatedAnnotationQueues'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
post:
description: Create an annotation queue
operationId: annotationQueues_createQueue
tags:
- AnnotationQueues
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueue'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateAnnotationQueueRequest'
/api/public/annotation-queues/{queueId}:
get:
description: Get an annotation queue by ID
operationId: annotationQueues_getQueue
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueue'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/annotation-queues/{queueId}/items:
get:
description: Get items for a specific annotation queue
operationId: annotationQueues_listQueueItems
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
- name: status
in: query
description: Filter by status
required: false
schema:
$ref: '#/components/schemas/AnnotationQueueStatus'
nullable: true
- name: page
in: query
description: page number, starts at 1
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
description: limit of items per page
required: false
schema:
type: integer
nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/PaginatedAnnotationQueueItems'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
post:
description: Add an item to an annotation queue
operationId: annotationQueues_createQueueItem
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueueItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateAnnotationQueueItemRequest'
/api/public/annotation-queues/{queueId}/items/{itemId}:
get:
description: Get a specific item from an annotation queue
operationId: annotationQueues_getQueueItem
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
- name: itemId
in: path
description: The unique identifier of the annotation queue item
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueueItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
patch:
description: Update an annotation queue item
operationId: annotationQueues_updateQueueItem
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
- name: itemId
in: path
description: The unique identifier of the annotation queue item
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueueItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateAnnotationQueueItemRequest'
delete:
description: Remove an item from an annotation queue
operationId: annotationQueues_deleteQueueItem
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
- name: itemId
in: path
description: The unique identifier of the annotation queue item
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DeleteAnnotationQueueItemResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/annotation-queues/{queueId}/assignments:
post:
description: Create an assignment for a user to an annotation queue
operationId: annotationQueues_createQueueAssignment
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/CreateAnnotationQueueAssignmentResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueueAssignmentRequest'
delete:
description: Delete an assignment for a user to an annotation queue
operationId: annotationQueues_deleteQueueAssignment
tags:
- AnnotationQueues
parameters:
- name: queueId
in: path
description: The unique identifier of the annotation queue
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DeleteAnnotationQueueAssignmentResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/AnnotationQueueAssignmentRequest'
/api/public/integrations/blob-storage:
get:
description: >-
Get all blob storage integrations for the organization (requires
organization-scoped API key)
operationId: blobStorageIntegrations_getBlobStorageIntegrations
tags:
- BlobStorageIntegrations
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/BlobStorageIntegrationsResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
put:
description: >-
Create or update a blob storage integration for a specific project
(requires organization-scoped API key). The configuration is validated
by performing a test upload to the bucket.
operationId: blobStorageIntegrations_upsertBlobStorageIntegration
tags:
- BlobStorageIntegrations
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/BlobStorageIntegrationResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateBlobStorageIntegrationRequest'
/api/public/integrations/blob-storage/{id}:
get:
description: >-
Get the sync status of a blob storage integration by integration ID
(requires organization-scoped API key)
operationId: blobStorageIntegrations_getBlobStorageIntegrationStatus
tags:
- BlobStorageIntegrations
parameters:
- name: id
in: path
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/BlobStorageIntegrationStatusResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
delete:
description: >-
Delete a blob storage integration by ID (requires organization-scoped
API key)
operationId: blobStorageIntegrations_deleteBlobStorageIntegration
tags:
- BlobStorageIntegrations
parameters:
- name: id
in: path
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/BlobStorageIntegrationDeletionResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/comments:
post:
description: >-
Create a comment. Comments may be attached to different object types
(trace, observation, session, prompt).
operationId: comments_create
tags:
- Comments
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/CreateCommentResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateCommentRequest'
get:
description: Get all comments
operationId: comments_get
tags:
- Comments
parameters:
- name: page
in: query
description: Page number, starts at 1.
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
description: >-
Limit of items per page. If you encounter api issues due to too
large page sizes, try to reduce the limit
required: false
schema:
type: integer
nullable: true
- name: objectType
in: query
description: >-
Filter comments by object type (trace, observation, session,
prompt).
required: false
schema:
type: string
nullable: true
- name: objectId
in: query
description: >-
Filter comments by object id. If objectType is not provided, an
error will be thrown.
required: false
schema:
type: string
nullable: true
- name: authorUserId
in: query
description: Filter comments by author user id.
required: false
schema:
type: string
nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/GetCommentsResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/comments/{commentId}:
get:
description: Get a comment by id
operationId: comments_get-by-id
tags:
- Comments
parameters:
- name: commentId
in: path
description: The unique langfuse identifier of a comment
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/Comment'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/dataset-items:
post:
description: Create a dataset item
operationId: datasetItems_create
tags:
- DatasetItems
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DatasetItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateDatasetItemRequest'
get:
description: >-
Get dataset items. Optionally specify a version to get the items as they
existed at that point in time.
Note: If version parameter is provided, datasetName must also be
provided.
operationId: datasetItems_list
tags:
- DatasetItems
parameters:
- name: datasetName
in: query
required: false
schema:
type: string
nullable: true
- name: sourceTraceId
in: query
required: false
schema:
type: string
nullable: true
- name: sourceObservationId
in: query
required: false
schema:
type: string
nullable: true
- name: version
in: query
description: >-
ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g.,
"2026-01-21T14:35:42Z").
If provided, returns state of dataset at this timestamp.
If not provided, returns the latest version. Requires datasetName to
be specified.
required: false
schema:
type: string
format: date-time
nullable: true
- name: page
in: query
description: page number, starts at 1
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
description: limit of items per page
required: false
schema:
type: integer
nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/PaginatedDatasetItems'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/dataset-items/{id}:
get:
description: Get a dataset item
operationId: datasetItems_get
tags:
- DatasetItems
parameters:
- name: id
in: path
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DatasetItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
delete:
description: >-
Delete a dataset item and all its run items. This action is
irreversible.
operationId: datasetItems_delete
tags:
- DatasetItems
parameters:
- name: id
in: path
required: true
schema:
type: string
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DeleteDatasetItemResponse'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/dataset-run-items:
post:
description: Create a dataset run item
operationId: datasetRunItems_create
tags:
- DatasetRunItems
parameters: []
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/DatasetRunItem'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateDatasetRunItemRequest'
get:
description: List dataset run items
operationId: datasetRunItems_list
tags:
- DatasetRunItems
parameters:
- name: datasetId
in: query
required: true
schema:
type: string
- name: runName
in: query
required: true
schema:
type: string
- name: page
in: query
description: page number, starts at 1
required: false
schema:
type: integer
nullable: true
- name: limit
in: query
description: limit of items per page
required: false
schema:
type: integer
nullable: true
responses:
'200':
description: ''
content:
application/json:
schema:
$ref: '#/components/schemas/PaginatedDatasetRunItems'
'400':
description: ''
content:
application/json:
schema: {}
'401':
description: ''
content:
application/json:
schema: {}
'403':
description: ''
content:
application/json:
schema: {}
'404':
description: ''
content:
application/json:
schema: {}
'405':
description: ''
content:
application/json:
schema: {}
security:
- BasicAuth: []
/api/public/v2/datasets:
get:
description: Get all datasets
operationId: datasets_list
tags:
- Datasets
parameters:
- name: page
in: query
description: page number, starts at 1
# --- truncated at 32 KB (376 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/langfuse/refs/heads/main/openapi/langfuse-openapi.yml