LangWatch

LangWatch Evaluators API

Configure and manage scorer evaluators — RAG quality (RAGAS), safety (Azure Content Safety, OpenAI Moderation, PII), format validation, semantic similarity, language detection, and LLM-as-Judge variants — that grade traces and dataset records.

LangWatch Evaluators API is one of 20 APIs that LangWatch publishes on the APIs.io network, described by a machine-readable OpenAPI specification.

This API exposes 1 machine-runnable capability that can be deployed as REST, MCP, or Agent Skill surfaces via Naftiko.

Tagged areas include Evaluations, Evaluators, RAGAS, and Safety. The published artifact set on APIs.io includes API documentation, an OpenAPI specification, and 1 Naftiko capability spec.

GitHub OpenAPI

Documentation

📖
Documentation
https://langwatch.ai/docs/api-reference/evaluators

Specifications

⚙
OpenAPI
https://raw.githubusercontent.com/api-evangelist/langwatch/refs/heads/main/openapi/langwatch-openapi.json

Other Resources

🔗
NaftikoCapability
https://raw.githubusercontent.com/api-evangelist/langwatch/refs/heads/main/capabilities/evaluators.yaml

OpenAPI Specification

langwatch-openapi.json Raw ↑ 
{
  "openapi": "3.1.0",
  "info": {
    "title": "LangWatch API",
    "version": "1.0.0",
    "description": "LangWatch openapi spec"
  },
  "servers": [
    {
      "url": "https://app.langwatch.ai"
    }
  ],
  "security": [
    {
      "project_api_key": []
    }
  ],
  "paths": {
    "/": {
      "get": {
        "responses": {
          "200": {
            "description": "Success",
            "content": {
              "application/json": {
                "schema": {
                  "type": "array",
                  "items": {
                    "type": "object",
                    "properties": {
                      "id": {
                        "type": "string"
                      },
                      "handle": {
                        "type": [
                          "string",
                          "null"
                        ]
                      },
                      "scope": {
                        "type": "string",
                        "enum": [
                          "ORGANIZATION",
                          "PROJECT"
                        ]
                      },
                      "name": {
                        "type": "string"
                      },
                      "updatedAt": {
                        "type": "string"
                      },
                      "projectId": {
                        "type": "string"
                      },
                      "organizationId": {
                        "type": "string"
                      },
                      "version": {
                        "type": "number"
                      },
                      "versionId": {
                        "type": "string"
                      },
                      "versionCreatedAt": {
                        "type": "string"
                      },
                      "model": {
                        "type": "string"
                      },
                      "prompt": {
                        "type": "string"
                      },
                      "messages": {
                        "type": "array",
                        "items": {
                          "type": "object",
                          "properties": {
                            "role": {
                              "type": "string",
                              "enum": [
                                "user",
                                "assistant",
                                "system"
                              ]
                            },
                            "content": {
                              "type": "string"
                            }
                          },
                          "required": [
                            "role",
                            "content"
                          ]
                        }
                      },
                      "response_format": {
                        "type": [
                          "object",
                          "null"
                        ],
                        "properties": {
                          "type": {
                            "type": "string",
                            "enum": [
                              "json_schema"
                            ]
                          },
                          "json_schema": {
                            "type": "object",
                            "properties": {
                              "name": {
                                "type": "string"
                              },
                              "schema": {
                                "type": "object"
                              }
                            },
                            "required": [
                              "name",
                              "schema"
                            ]
                          }
                        },
                        "required": [
                          "type",
                          "json_schema"
                        ]
                      }
                    },
                    "required": [
                      "id",
                      "handle",
                      "scope",
                      "name",
                      "updatedAt",
                      "projectId",
                      "organizationId",
                      "version",
                      "versionId",
                      "versionCreatedAt",
                      "model",
                      "prompt",
                      "messages",
                      "response_format"
                    ]
                  }
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          }
        },
        "operationId": "getIndex",
        "parameters": [],
        "description": "Get all prompts for a project"
      },
      "post": {
        "responses": {
          "200": {
            "description": "Success",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "id": {
                      "type": "string"
                    },
                    "handle": {
                      "type": [
                        "string",
                        "null"
                      ]
                    },
                    "scope": {
                      "type": "string",
                      "enum": [
                        "ORGANIZATION",
                        "PROJECT"
                      ]
                    },
                    "name": {
                      "type": "string"
                    },
                    "updatedAt": {
                      "type": "string"
                    },
                    "projectId": {
                      "type": "string"
                    },
                    "organizationId": {
                      "type": "string"
                    },
                    "version": {
                      "type": "number"
                    },
                    "versionId": {
                      "type": "string"
                    },
                    "versionCreatedAt": {
                      "type": "string"
                    },
                    "model": {
                      "type": "string"
                    },
                    "prompt": {
                      "type": "string"
                    },
                    "messages": {
                      "type": "array",
                      "items": {
                        "type": "object",
                        "properties": {
                          "role": {
                            "type": "string",
                            "enum": [
                              "user",
                              "assistant",
                              "system"
                            ]
                          },
                          "content": {
                            "type": "string"
                          }
                        },
                        "required": [
                          "role",
                          "content"
                        ]
                      }
                    },
                    "response_format": {
                      "type": [
                        "object",
                        "null"
                      ],
                      "properties": {
                        "type": {
                          "type": "string",
                          "enum": [
                            "json_schema"
                          ]
                        },
                        "json_schema": {
                          "type": "object",
                          "properties": {
                            "name": {
                              "type": "string"
                            },
                            "schema": {
                              "type": "object"
                            }
                          },
                          "required": [
                            "name",
                            "schema"
                          ]
                        }
                      },
                      "required": [
                        "type",
                        "json_schema"
                      ]
                    }
                  },
                  "required": [
                    "id",
                    "handle",
                    "scope",
                    "name",
                    "updatedAt",
                    "projectId",
                    "organizationId",
                    "version",
                    "versionId",
                    "versionCreatedAt",
                    "model",
                    "prompt",
                    "messages",
                    "response_format"
                  ]
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          }
        },
        "operationId": "postIndex",
        "parameters": [],
        "description": "Create a new prompt with default initial version",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "type": "object",
                "properties": {
                  "handle": {
                    "type": "string",
                    "pattern": "^[a-z0-9_-]+(?:\\/[a-z0-9_-]+)?$"
                  },
                  "scope": {
                    "type": "string",
                    "enum": [
                      "ORGANIZATION",
                      "PROJECT"
                    ],
                    "default": "PROJECT"
                  },
                  "authorId": {
                    "type": "string"
                  },
                  "prompt": {
                    "type": "string"
                  },
                  "messages": {
                    "type": "array",
                    "items": {
                      "type": "object",
                      "properties": {
                        "role": {
                          "type": "string",
                          "enum": [
                            "user",
                            "assistant",
                            "system"
                          ]
                        },
                        "content": {
                          "type": "string"
                        }
                      },
                      "required": [
                        "role",
                        "content"
                      ]
                    }
                  },
                  "inputs": {
                    "type": "array",
                    "items": {
                      "type": "object",
                      "properties": {
                        "identifier": {
                          "type": "string",
                          "minLength": 1
                        },
                        "type": {
                          "type": "string",
                          "enum": [
                            "str",
                            "float",
                            "bool",
                            "image",
                            "list[str]",
                            "list[float]",
                            "list[int]",
                            "list[bool]",
                            "dict"
                          ]
                        }
                      },
                      "required": [
                        "identifier",
                        "type"
                      ]
                    }
                  },
                  "outputs": {
                    "type": "array",
                    "items": {
                      "type": "object",
                      "properties": {
                        "identifier": {
                          "type": "string",
                          "minLength": 1
                        },
                        "type": {
                          "type": "string",
                          "enum": [
                            "str",
                            "float",
                            "bool",
                            "json_schema"
                          ]
                        },
                        "json_schema": {
                          "type": "object",
                          "properties": {
                            "type": {
                              "type": "string",
                              "minLength": 1
                            }
                          },
                          "required": [
                            "type"
                          ],
                          "additionalProperties": true
                        }
                      },
                      "required": [
                        "identifier",
                        "type"
                      ]
                    }
                  }
                },
                "required": [
                  "handle"
                ]
              }
            }
          }
        }
      }
    },
    "/api/agents": {
      "get": {
        "responses": {},
        "operationId": "getApiAgents",
        "parameters": [
          {
            "in": "query",
            "name": "page",
            "schema": {
              "type": "integer",
              "exclusiveMinimum": 0,
              "default": 1
            }
          },
          {
            "in": "query",
            "name": "limit",
            "schema": {
              "type": "integer",
              "exclusiveMinimum": 0,
              "maximum": 1000,
              "default": 50
            }
          }
        ],
        "description": "List all non-archived agents for the project (paginated)"
      },
      "post": {
        "responses": {},
        "operationId": "postApiAgents",
        "parameters": [],
        "description": "Create a new agent",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "type": "object",
                "properties": {
                  "name": {
                    "type": "string",
                    "minLength": 1,
                    "maxLength": 255
                  },
                  "type": {
                    "type": "string",
                    "enum": [
                      "signature",
                      "code",
                      "workflow",
                      "http"
                    ]
                  },
                  "config": {
                    "type": "object",
                    "additionalProperties": {}
                  },
                  "workflowId": {
                    "type": "string"
                  }
                },
                "required": [
                  "name",
                  "type",
                  "config"
                ]
              }
            }
          }
        }
      }
    },
    "/api/agents/{id}": {
      "get": {
        "responses": {},
        "operationId": "getApiAgentsById",
        "parameters": [
          {
            "schema": {
              "type": "string"
            },
            "in": "path",
            "name": "id",
            "required": true
          }
        ],
        "description": "Get an agent by its id"
      },
      "patch": {
        "responses": {},
        "operationId": "patchApiAgentsById",
        "parameters": [
          {
            "schema": {
              "type": "string"
            },
            "in": "path",
            "name": "id",
            "required": true
          }
        ],
        "description": "Update an agent by its id",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "type": "object",
                "properties": {
                  "name": {
                    "type": "string",
                    "minLength": 1,
                    "maxLength": 255
                  },
                  "type": {
                    "type": "string",
                    "enum": [
                      "signature",
                      "code",
                      "workflow",
                      "http"
                    ]
                  },
                  "config": {
                    "type": "object",
                    "additionalProperties": {}
                  },
                  "workflowId": {
                    "type": [
                      "string",
                      "null"
                    ]
                  }
                }
              }
            }
          }
        }
      },
      "delete": {
        "responses": {},
        "operationId": "deleteApiAgentsById",
        "parameters": [
          {
            "schema": {
              "type": "string"
            },
            "in": "path",
            "name": "id",
            "required": true
          }
        ],
        "description": "Archive an agent (soft-delete)"
      }
    },
    "/api/analytics/timeseries": {
      "post": {
        "responses": {
          "200": {
            "description": "Timeseries analytics data with current and previous periods",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "currentPeriod": {
                      "type": "array",
                      "items": {
                        "type": "object",
                        "additionalProperties": {}
                      }
                    },
                    "previousPeriod": {
                      "type": "array",
                      "items": {
                        "type": "object",
                        "additionalProperties": {}
                      }
                    }
                  },
                  "required": [
                    "currentPeriod",
                    "previousPeriod"
                  ]
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    },
                    "message": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    },
                    "message": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "422": {
            "description": "Unprocessable Entity",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    },
                    "message": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "type": "object",
                  "properties": {
                    "error": {
                      "type": "string"
                    },
                    "message": {
                      "type": "string"
                    }
                  },
                  "required": [
                    "error"
                  ]
                }
              }
            }
          }
        },
        "operationId": "postApiAnalyticsTimeseries",
        "parameters": [],
        "description": "Query analytics timeseries data with metrics, aggregations, and filters",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "type": "object",
                "properties": {
                  "startDate": {
                    "anyOf": [
                      {
                        "type": "number"
                      },
                      {
                        "type": "string"
                      }
                    ]
                  },
                  "endDate": {
                    "anyOf": [
                      {
                        "type": "number"
                      },
                      {
                        "type": "string"
                      }
                    ]
                  },
                  "query": {
                    "type": "string"
                  },
                  "filters": {
                    "type": "object",
                    "properties": {
                      "topics.topics": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "topics.subtopics": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "metadata.user_id": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "metadata.thread_id": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "metadata.customer_id": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "metadata.labels": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
                                  "type": "string"
                                }
                              }
                            }
                          }
                        ]
                      },
                      "metadata.key": {
                        "anyOf": [
                          {
                            "type": "array",
                            "items": {
                              "type": "string"
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "array",
                              "items": {
                                "type": "string"
                              }
                            }
                          },
                          {
                            "type": "object",
                            "additionalProperties": {
                              "type": "object",
                              "additionalProperties": {
                                "type": "array",
                                "items": {
           

# --- truncated at 32 KB (922 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/langwatch/refs/heads/main/openapi/langwatch-openapi.json