From f2bdee6726ff49031a66f9b76a38d92e7af59d01 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:37:30 -0500 Subject: [PATCH] feat(ai): add gateway evaluation providers (#50665) --- packages/ai/README.md | 12 +- .../ai/src/experimental/evaluation-client.ts | 64 ++++- packages/ai/src/experimental/evaluation.ts | 6 +- packages/ai/src/experimental/system-one.ts | 244 ++++++++---------- packages/ai/src/providers/index.ts | 1 + packages/ai/src/providers/openrouter.ts | 23 +- .../ai/src/providers/vercel-ai-gateway.ts | 148 +++++++++++ packages/ai/test/evaluation.test.ts | 131 +++++++++- packages/ai/test/evaluation.types.ts | 30 ++- packages/ai/test/exports.test.ts | 3 + ...es-choice-score-and-boolean-questions.json | 33 +++ .../test/provider/evaluation.recorded.test.ts | 29 ++- 12 files changed, 564 insertions(+), 160 deletions(-) create mode 100644 packages/ai/src/providers/vercel-ai-gateway.ts create mode 100644 packages/ai/test/fixtures/recordings/openrouter-evaluation/evaluates-choice-score-and-boolean-questions.json diff --git a/packages/ai/README.md b/packages/ai/README.md index 6940daa5bca..577af276af4 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -66,7 +66,17 @@ console.log(response.answers.refund.probability) ``` `TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads -`OPENCODE_API_KEY`. The common API uses `boolean`; System One routes lower it to native `noul`. +`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape: + +```ts +import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers" + +OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13") +VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev") +``` + +OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`. +The common API uses `boolean`; System One routes lower it to native `noul`. Choice and score confidence plus score legends remain available in provider metadata, and the provider's rounded probabilities are returned unchanged. diff --git a/packages/ai/src/experimental/evaluation-client.ts b/packages/ai/src/experimental/evaluation-client.ts index b6be4722efc..8fdd2ceb410 100644 --- a/packages/ai/src/experimental/evaluation-client.ts +++ b/packages/ai/src/experimental/evaluation-client.ts @@ -1,6 +1,6 @@ import { Context, Effect, Layer } from "effect" import { RequestExecutor } from "../route/executor.js" -import { mergeHttpOptions, type AIError } from "../schema/index.js" +import { AIError, InvalidProviderOutputError, mergeHttpOptions } from "../schema/index.js" import { sanitizeSurrogates } from "../utils/sanitize.js" import { type EvaluationOptions, @@ -30,17 +30,59 @@ export const layer: Layer.Layer = Layer const executor = yield* RequestExecutor.Service return Service.of({ evaluate: (request) => - request.model.route.evaluate( - { - ...sanitizeSurrogates({ - ...request, - model: undefined, - http: mergeHttpOptions(request.model.http, request.http), + request.model.route + .evaluate( + { + ...sanitizeSurrogates({ + ...request, + model: undefined, + http: mergeHttpOptions(request.model.http, request.http), + }), + model: request.model, + }, + executor.execute, + ) + .pipe( + Effect.flatMap((response) => { + const questions = Object.entries(request.questions) + if ( + questions.length === Object.keys(response.answers).length && + questions.every(([id, question]) => { + const answer = response.answers[id] + if (question.type === "boolean") return answer?.type === "boolean" + if (question.type === "choice") { + if (answer?.type !== "choice" || !Object.hasOwn(question.criteria, answer.choice)) return false + if (answer.probabilities === undefined) return true + const keys = Object.keys(question.criteria) + const probabilities = answer.probabilities + return ( + Object.keys(probabilities).length === keys.length && + keys.every((key) => Object.hasOwn(probabilities, key)) + ) + } + if (answer?.type !== "score" || answer.score < 0 || answer.score > question.criteria.length - 1) + return false + if (answer.probabilities === undefined) return true + const keys = question.criteria.map((_, index) => String(index)) + const probabilities = answer.probabilities + return ( + Object.keys(probabilities).length === keys.length && + keys.every((key) => Object.hasOwn(probabilities, key)) + ) + }) + ) + return Effect.succeed(response as EvaluationResponseFor) + return Effect.fail( + new AIError({ + reason: new InvalidProviderOutputError({ + route: request.model.route.id, + message: "Evaluation answers do not match the requested questions", + cause: response.answers, + }), + }), + ) }), - model: request.model, - }, - executor.execute, - ), + ), }) }), ) diff --git a/packages/ai/src/experimental/evaluation.ts b/packages/ai/src/experimental/evaluation.ts index 269f2f052a5..fedda1100e9 100644 --- a/packages/ai/src/experimental/evaluation.ts +++ b/packages/ai/src/experimental/evaluation.ts @@ -105,10 +105,10 @@ export type EvaluationOptions = Record export interface EvaluationRoute { readonly id: string - readonly evaluate: ( - request: EvaluationRequestFor, + readonly evaluate: ( + request: EvaluationRequestFor, execute: Execute, - ) => Effect.Effect, AIError> + ) => Effect.Effect } export class EvaluationModel { diff --git a/packages/ai/src/experimental/system-one.ts b/packages/ai/src/experimental/system-one.ts index 0ea8085d5c8..591d6cd3117 100644 --- a/packages/ai/src/experimental/system-one.ts +++ b/packages/ai/src/experimental/system-one.ts @@ -1,17 +1,14 @@ import { Effect, Schema } from "effect" import { Headers, HttpClientRequest } from "effect/unstable/http" import { - BooleanAnswer, - ChoiceAnswer, ChoiceQuestion, EvaluationInput, EvaluationModel, EvaluationResponse, EvaluationRounding, - ScoreAnswer, ScoreQuestion, - type EvaluationResponseFor, - type EvaluationRoute, + type EvaluationAnswer, + type EvaluationOptions, } from "./evaluation.js" import { Auth, type Definition as AuthDefinition } from "../route/auth.js" import { @@ -72,15 +69,23 @@ const Score = Schema.Struct({ legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)), confidence: Schema.optional(Probability), }) -const NativeUsage = Schema.Struct({ - input_tokens: Schema.optional(Schema.Number), - output_tokens: Schema.optional(Schema.Number), +const Answer = Schema.Union([NoulAnswer, Choice, Score]).pipe(Schema.toTaggedUnion("type")) +const NativeUsage = Schema.StructWithRest( + Schema.Struct({ + input_tokens: Schema.optional(Schema.Number), + output_tokens: Schema.optional(Schema.Number), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) +const Response = Schema.Struct({ + model: Schema.String, + answers: Schema.Record(Schema.String, Answer), + usage: Schema.optional(NativeUsage), + id: Schema.optional(Schema.String), + provider: Schema.optional(Schema.String), + provider_metadata: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))), }) -const encode = Schema.encodeUnknownEffect(Schema.fromJsonString(Request)) -const exact = (x: Readonly>, keys: ReadonlyArray) => - Object.keys(x).length === keys.length && keys.every((key) => Object.hasOwn(x, key)) - export interface ModelInput { readonly id: string | ModelID readonly provider: string @@ -91,132 +96,99 @@ export interface ModelInput { readonly http?: HttpOptions } -export const model = (cfg: ModelInput) => { - const route: EvaluationRoute = { - id: "system-one", - evaluate: (req, send) => - Effect.gen(function* () { - const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`) - Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value)) - const body = yield* encode({ - ...mergeJsonRecords(req.options, req.http?.body), - model: req.model.id, - state: req.state, - questions: Object.fromEntries( - Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]), - ), - }).pipe( - Effect.mapError( - (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }), - ), - ) - const headers = yield* Auth.toEffect(cfg.auth)({ - request: req, - method: "POST", - url: url.toString(), - body, - headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }), - }) - const res = yield* send( - HttpClientRequest.post(url).pipe( - HttpClientRequest.setHeaders(headers), - HttpClientRequest.bodyText(body, "application/json"), - ), - ) - const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers }) - const fail = (message: string, cause: unknown, body?: string) => - new AIError({ reason: new InvalidProviderOutputError({ route: route.id, message, body, http, cause }) }) - const text = yield* res.text.pipe( - Effect.mapError((cause) => fail("Failed to read the System One response", cause)), - ) - const entries = Object.entries(req.questions) - const output = Schema.Struct({ - model: Schema.String, - answers: Schema.Struct( - Object.fromEntries( - entries.map(([id, question]) => { - if (question.type === "boolean") return [id, NoulAnswer] - if (question.type === "choice") { - const keys = Object.keys(question.criteria) - return [ - id, - Choice.pipe( - Schema.refine( - (x): x is typeof x => - Object.hasOwn(question.criteria, x.choice) && exact(x.probabilities, keys), - { message: `Question "${id}" returned an invalid choice answer` }, - ), - ), - ] - } - const keys = question.criteria.map((_, index) => String(index)) +export const model = (cfg: ModelInput) => + EvaluationModel.make({ + id: cfg.id, + provider: cfg.provider, + http: cfg.http, + route: { + id: "system-one", + evaluate: (req, send) => + Effect.gen(function* () { + const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`) + Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value)) + const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({ + ...mergeJsonRecords(req.options, req.http?.body), + model: req.model.id, + state: req.state, + questions: Object.fromEntries( + Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]), + ), + }).pipe( + Effect.mapError( + (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }), + ), + ) + const headers = yield* Auth.toEffect(cfg.auth)({ + request: req, + method: "POST", + url: url.toString(), + body, + headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }), + }) + const res = yield* send( + HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders(headers), + HttpClientRequest.bodyText(body, "application/json"), + ), + ) + const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers }) + const fail = (message: string, cause: unknown, body?: string) => + new AIError({ reason: new InvalidProviderOutputError({ route: "system-one", message, body, http, cause }) }) + const text = yield* res.text.pipe( + Effect.mapError((cause) => fail("Failed to read the System One response", cause)), + ) + const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe( + Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)), + ) + + const confidence: Record = {} + const legend: Record> = {} + const answers = Object.fromEntries( + Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => { + if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }] + if (answer.type === "choice") { + if (answer.confidence !== undefined) confidence[id] = answer.confidence return [ id, - Score.pipe( - Schema.refine( - (x): x is typeof x => - x.score >= 0 && x.score <= question.criteria.length - 1 && exact(x.probabilities, keys), - { message: `Question "${id}" returned an invalid score answer` }, - ), - ), + { + type: "choice", + choice: answer.choice, + probabilities: answer.probabilities, + }, ] - }), - ) as Record>, - ), - usage: Schema.optional(NativeUsage), - }) - const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(output))(text).pipe( - Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)), - ) - - const confidence: Record = {} - const legend: Record> = {} - const answers = Object.fromEntries( - entries.map(([id, question]) => { - const answer = data.answers[id] - if (question.type === "boolean") - return [ - id, - { type: "boolean", probability: (answer as typeof NoulAnswer.Type).noul } satisfies BooleanAnswer, - ] - if (question.type === "choice") { - const value = answer as typeof Choice.Type - if (value.confidence !== undefined) confidence[id] = value.confidence - return [ - id, - { type: "choice", choice: value.choice, probabilities: value.probabilities } satisfies ChoiceAnswer, - ] - } - const value = answer as typeof Score.Type - if (value.confidence !== undefined) confidence[id] = value.confidence - if (value.legend !== undefined) legend[id] = value.legend - return [id, { type: "score", score: value.score, probabilities: value.probabilities } satisfies ScoreAnswer] - }), - ) - const meta = { - ...(Object.keys(confidence).length === 0 ? {} : { confidence }), - ...(Object.keys(legend).length === 0 ? {} : { legend }), - } - return new EvaluationResponse({ - model: ModelID.make(data.model), - answers, - usage: data.usage - ? new Usage({ - inputTokens: data.usage.input_tokens, - outputTokens: data.usage.output_tokens, - totalTokens: - data.usage.input_tokens === undefined && data.usage.output_tokens === undefined - ? undefined - : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0), - providerMetadata: { [cfg.providerMetadataKey]: data.usage }, - }) - : undefined, - rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), - providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta }, - }) as EvaluationResponseFor - }), - } - return EvaluationModel.make({ id: cfg.id, provider: cfg.provider, route, http: cfg.http }) -} + } + if (answer.confidence !== undefined) confidence[id] = answer.confidence + if (answer.legend !== undefined) legend[id] = answer.legend + return [id, { type: "score", score: answer.score, probabilities: answer.probabilities }] + }), + ) + const meta = { + ...(data.id === undefined ? {} : { responseId: data.id }), + ...(data.provider === undefined ? {} : { provider: data.provider }), + ...data.provider_metadata?.[cfg.providerMetadataKey], + ...(Object.keys(confidence).length === 0 ? {} : { confidence }), + ...(Object.keys(legend).length === 0 ? {} : { legend }), + } + return new EvaluationResponse({ + model: ModelID.make(data.model), + answers, + usage: data.usage + ? new Usage({ + inputTokens: data.usage.input_tokens, + outputTokens: data.usage.output_tokens, + totalTokens: + data.usage.input_tokens === undefined && data.usage.output_tokens === undefined + ? undefined + : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0), + providerMetadata: { [cfg.providerMetadataKey]: data.usage }, + }) + : undefined, + rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }), + providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta }, + }) + }), + }, + }) export const SystemOne = { model } as const diff --git a/packages/ai/src/providers/index.ts b/packages/ai/src/providers/index.ts index 28cb2c361e6..145cc69d0ba 100644 --- a/packages/ai/src/providers/index.ts +++ b/packages/ai/src/providers/index.ts @@ -28,6 +28,7 @@ export * as OpenCodeZen from "./opencode-zen.js" export * as OpenRouter from "./openrouter.js" export * as TogetherAI from "./togetherai.js" export * as TypeSafeAI from "./typesafe-ai.js" +export * as VercelAIGateway from "./vercel-ai-gateway.js" export * as XAI from "./xai.js" export * as ZAI from "./zai.js" export * as ZAICodingPlan from "./zai-coding-plan.js" diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index 4bec70e3601..29f8d58e225 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -3,8 +3,9 @@ import { Route, type RouteDefaultsInput } from "../route/client.js" import { Endpoint } from "../route/endpoint.js" import { Protocol } from "../route/protocol.js" import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" -import { ProviderID, type CacheHint, type ModelID } from "../schema/index.js" +import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js" import type { ProviderPackage } from "../provider-package.js" +import { SystemOne } from "../experimental/system-one.js" import { OpenAIChat } from "../protocols/openai-chat.js" import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js" import { isRecord } from "../protocols/shared.js" @@ -71,6 +72,14 @@ export interface OpenRouterOptions { export type OpenRouterProviderOptionsInput = OpenRouterOptions +export interface OpenRouterEvaluationOptions { + readonly [key: string]: unknown + readonly provider?: OpenRouterProviderRouting + readonly session_id?: string + readonly trace?: Readonly> + readonly user?: string +} + export type LanguageModelOptions = Omit & ProviderAuthOption<"optional"> & { readonly baseURL?: string @@ -181,15 +190,27 @@ const configuredRoute = (input: LanguageModelOptions) => { export const configure = (input: LanguageModelOptions = {}) => { const route = configuredRoute(input) + const evaluation = (modelID: string | ModelID) => + SystemOne.model({ + id: modelID, + provider: id, + providerMetadataKey: "openrouter", + auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"), + baseURL: input.baseURL ?? baseURL, + headers: input.headers, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + }) return { id, model: (modelID: string | ModelID) => route.model({ id: modelID, compatibility: { supportsPromptCacheKey: true } }), + experimental: { evaluation }, configure, } } export const provider = configure() +export const experimental = provider.experimental export const model: ProviderPackage.Definition["model"] = ( modelID, { apiKey, baseURL, body, headers, ...providerOptions }, diff --git a/packages/ai/src/providers/vercel-ai-gateway.ts b/packages/ai/src/providers/vercel-ai-gateway.ts new file mode 100644 index 00000000000..ff87c2a97d5 --- /dev/null +++ b/packages/ai/src/providers/vercel-ai-gateway.ts @@ -0,0 +1,148 @@ +import { Effect, Schema } from "effect" +import { Headers, HttpClientRequest } from "effect/unstable/http" +import { + EvaluationAnswer, + EvaluationInput, + EvaluationModel, + EvaluationQuestion, + EvaluationResponse, + EvaluationRounding, +} from "../experimental/evaluation.js" +import { Auth } from "../route/auth.js" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" +import { + AIError, + HttpContext, + HttpOptions, + InvalidProviderOutputError, + InvalidRequestError, + ModelID, + ProviderID, + ProviderMetadata, + Usage, +} from "../schema/index.js" + +export const id = ProviderID.make("vercel-ai-gateway") +const baseURL = "https://ai-gateway.vercel.sh/v1" + +export interface EvaluationOptions { + readonly [key: string]: unknown + readonly gateway?: Readonly<{ + readonly [key: string]: unknown + readonly zeroDataRetention?: boolean + readonly only?: ReadonlyArray + }> +} + +export type Options = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions.Input +} + +const Request = Schema.StructWithRest( + Schema.Struct({ + model: Schema.String, + state: EvaluationInput, + questions: Schema.Record(Schema.String, EvaluationQuestion), + providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + }), + [Schema.Record(Schema.String, Schema.Any)], +) +const Response = Schema.Struct({ + model: Schema.optional(Schema.String), + answers: Schema.Record(Schema.String, EvaluationAnswer), + usage: Schema.optional( + Schema.Struct({ + inputTokens: Schema.optional(Schema.Number), + outputTokens: Schema.optional(Schema.Number), + }), + ), + rounding: Schema.optional(EvaluationRounding), + providerMetadata: Schema.optional(ProviderMetadata), +}) + +export const configure = (input: Options = {}) => { + const evaluation = (modelID: string | ModelID) => + EvaluationModel.make({ + id: modelID, + provider: id, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), + route: { + id: "vercel-evaluation", + evaluate: (req, send) => + Effect.gen(function* () { + const url = new URL(`${(input.baseURL ?? baseURL).replace(/\/$/, "")}/evaluate`) + Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value)) + const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({ + ...req.http?.body, + model: req.model.id, + state: req.state, + questions: req.questions, + providerOptions: req.options, + }).pipe( + Effect.mapError( + (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }), + ), + ) + const headers = yield* Auth.toEffect( + AuthOptions.bearer(input, ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"]), + )({ + request: req, + method: "POST", + url: url.toString(), + body, + headers: Headers.fromInput({ ...input.headers, ...req.http?.headers }), + }) + const res = yield* send( + HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders(headers), + HttpClientRequest.bodyText(body, "application/json"), + ), + ) + const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers }) + const fail = (message: string, cause: unknown, body?: string) => + new AIError({ + reason: new InvalidProviderOutputError({ + route: "vercel-evaluation", + message, + body, + http, + cause, + }), + }) + const text = yield* res.text.pipe( + Effect.mapError((cause) => fail("Failed to read the Vercel AI Gateway evaluation response", cause)), + ) + const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe( + Effect.mapError((cause) => + fail("Vercel AI Gateway returned an invalid evaluation response", cause, text), + ), + ) + return new EvaluationResponse({ + model: ModelID.make(data.model ?? req.model.id), + answers: data.answers, + usage: data.usage + ? new Usage({ + inputTokens: data.usage.inputTokens, + outputTokens: data.usage.outputTokens, + totalTokens: + data.usage.inputTokens === undefined && data.usage.outputTokens === undefined + ? undefined + : (data.usage.inputTokens ?? 0) + (data.usage.outputTokens ?? 0), + providerMetadata: { gateway: data.usage }, + }) + : undefined, + rounding: data.rounding, + providerMetadata: data.providerMetadata, + }) + }), + }, + }) + return { id, experimental: { evaluation }, configure } +} + +export const provider = configure() +export const experimental = provider.experimental + +export * as VercelAIGateway from "./vercel-ai-gateway.js" diff --git a/packages/ai/test/evaluation.test.ts b/packages/ai/test/evaluation.test.ts index 55da51a68ba..a228e7465e9 100644 --- a/packages/ai/test/evaluation.test.ts +++ b/packages/ai/test/evaluation.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { HttpClientRequest } from "effect/unstable/http" import { Evaluation, EvaluationClient } from "../src/experimental.js" -import { OpenCodeZen, TypeSafeAI } from "../src/providers.js" +import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" import { it } from "./lib/effect.js" import { dynamicResponse } from "./lib/http.js" @@ -154,6 +154,135 @@ describe("experimental Evaluation", () => { ), ) + it.effect("evaluates through OpenRouter System One", () => + Evaluation.run({ + model: OpenRouter.configure({ + apiKey: "openrouter-key", + baseURL: "https://openrouter.test/api/v1", + }).experimental.evaluation("typesafe/jev-1.13"), + state: "refund", + questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } }, + options: { user: "user-1", session_id: "session-1" }, + }).pipe( + Effect.tap((response) => + Effect.sync(() => { + expect(response.answers.refund.probability).toBe(0.98) + expect(response.providerMetadata?.openrouter).toMatchObject({ responseId: "gen-1", provider: "TypeSafe" }) + expect(response.usage?.providerMetadata?.openrouter).toEqual({ + input_tokens: 10, + output_tokens: 2, + cost: 0.0001, + }) + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + expect(input.request.url).toBe("https://openrouter.test/api/v1/systemone") + expect(input.request.headers.authorization).toBe("Bearer openrouter-key") + expect(JSON.parse(input.text)).toMatchObject({ + model: "typesafe/jev-1.13", + user: "user-1", + session_id: "session-1", + questions: { refund: { type: "noul" } }, + }) + return Effect.succeed( + input.respond( + JSON.stringify({ + id: "gen-1", + model: "typesafe/jev-1.13-20260917", + provider: "TypeSafe", + answers: { refund: { type: "noul", noul: 0.98 } }, + usage: { input_tokens: 10, output_tokens: 2, cost: 0.0001 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ) + }), + ), + ), + ), + ), + ) + + it.effect("evaluates through Vercel AI Gateway", () => + Evaluation.run({ + model: VercelAIGateway.configure({ + apiKey: "gateway-key", + baseURL: "https://gateway.test/v1/", + }).experimental.evaluation("typesafe-ai/jev"), + state: "refund", + questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } }, + options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } }, + }).pipe( + Effect.tap((response) => + Effect.sync(() => { + expect(response.answers.refund.probability).toBe(0.98) + expect(response.usage?.totalTokens).toBe(12) + expect(response.providerMetadata?.gateway).toMatchObject({ generationId: "gen-1", cost: "0.0001" }) + }), + ), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + expect(input.request.url).toBe("https://gateway.test/v1/evaluate") + expect(input.request.headers.authorization).toBe("Bearer gateway-key") + expect(JSON.parse(input.text)).toEqual({ + model: "typesafe-ai/jev", + state: "refund", + questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } }, + providerOptions: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } }, + }) + return Effect.succeed( + input.respond( + JSON.stringify({ + model: "typesafe-ai/jev", + answers: { refund: { type: "boolean", probability: 0.98 } }, + usage: { inputTokens: 10, outputTokens: 2 }, + providerMetadata: { gateway: { generationId: "gen-1", cost: "0.0001" } }, + }), + { headers: { "content-type": "application/json" } }, + ), + ) + }), + ), + ), + ), + ), + ) + + it.effect("rejects answers that do not match their questions", () => + Evaluation.run({ + model: VercelAIGateway.configure({ + apiKey: "gateway-key", + baseURL: "https://gateway.test/v1", + }).experimental.evaluation("typesafe-ai/jev"), + state: "refund", + questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } }, + }).pipe( + Effect.flip, + Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))), + Effect.provide( + EvaluationClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.succeed( + input.respond( + JSON.stringify({ + answers: { refund: { type: "choice", choice: "yes", probabilities: { yes: 1 } } }, + }), + { headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + ), + ), + ), + ) + it.effect("rejects malformed questions before network I/O", () => Effect.gen(function* () { const error = yield* Evaluation.run({ diff --git a/packages/ai/test/evaluation.types.ts b/packages/ai/test/evaluation.types.ts index ba74de93481..9479cbc5c3d 100644 --- a/packages/ai/test/evaluation.types.ts +++ b/packages/ai/test/evaluation.types.ts @@ -1,7 +1,7 @@ import { Effect } from "effect" import { Evaluation, EvaluationClient, EvaluationModel, type EvaluationRoute } from "../src/experimental.js" import type { Service } from "../src/experimental/evaluation-client.js" -import { OpenCodeZen, TypeSafeAI } from "../src/providers.js" +import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js" type Requirements = T extends Effect.Effect ? R : never type Success = T extends Effect.Effect ? A : never @@ -58,3 +58,31 @@ Evaluation.run({ questions: { ok: { type: "boolean", instructions: "OK?" } }, options: { temperature: "high" }, }) + +Evaluation.run({ + model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"), + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { provider: { zdr: true }, session_id: "session-1", user: "user-1" }, +}) +// @ts-expect-error OpenRouter session IDs are strings. +Evaluation.run({ + model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"), + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { session_id: 1 }, +}) + +Evaluation.run({ + model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"), + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } }, +}) +// @ts-expect-error Vercel zero-data-retention controls are boolean. +Evaluation.run({ + model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"), + state: "hello", + questions: { ok: { type: "boolean", instructions: "OK?" } }, + options: { gateway: { zeroDataRetention: "yes" } }, +}) diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index ce0cd4a7175..497b8ba610d 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -13,6 +13,7 @@ import { OpenAICompatible, OpenRouter, TypeSafeAI, + VercelAIGateway, XAI, } from "@opencode/ai/providers" import { @@ -72,8 +73,10 @@ describe("public exports", () => { expect(CloudflareWorkersAI.configure).toBeFunction() expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction() expect(OpenRouter.model).toBeFunction() + expect(OpenRouter.experimental.evaluation).toBeFunction() expect(TypeSafeAI.experimental.evaluation).toBeFunction() expect(OpenCodeZen.experimental.evaluation).toBeFunction() + expect(VercelAIGateway.experimental.evaluation).toBeFunction() expect(XAI.model).toBeFunction() expect(XAI.provider.responses).toBe(XAI.responses) expect(XAI.provider.chat).toBe(XAI.chat) diff --git a/packages/ai/test/fixtures/recordings/openrouter-evaluation/evaluates-choice-score-and-boolean-questions.json b/packages/ai/test/fixtures/recordings/openrouter-evaluation/evaluates-choice-score-and-boolean-questions.json new file mode 100644 index 00000000000..32749cdcc93 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/openrouter-evaluation/evaluates-choice-score-and-boolean-questions.json @@ -0,0 +1,33 @@ +{ + "version": 1, + "metadata": { + "model": "typesafe/jev-1.13", + "tags": [ + "prefix:openrouter-evaluation", + "provider:openrouter", + "protocol:system-one" + ], + "name": "openrouter-evaluation/evaluates-choice-score-and-boolean-questions", + "recordedAt": "2026-09-22T14:54:46.656Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://openrouter.ai/api/v1/systemone", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"typesafe/jev-1.13\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"typesafe/jev-1.13-20260917\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"probabilities\":{\"billing\":1,\"sales\":0,\"technical\":0},\"confidence\":1},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.01,\"1\":0.96,\"2\":0.03},\"confidence\":0.95},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69,\"cost\":0.000017724},\"id\":\"gen-dec-1790088886-JRlffOLUwQdx4Lltu2se\",\"provider\":\"TypeSafe\"}" + } + } + ] +} diff --git a/packages/ai/test/provider/evaluation.recorded.test.ts b/packages/ai/test/provider/evaluation.recorded.test.ts index 43c51e3466c..94bf216e394 100644 --- a/packages/ai/test/provider/evaluation.recorded.test.ts +++ b/packages/ai/test/provider/evaluation.recorded.test.ts @@ -1,7 +1,7 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { Evaluation } from "../../src/experimental.js" -import { OpenCodeZen, TypeSafeAI } from "../../src/providers.js" +import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js" +import { OpenCodeZen, OpenRouter, TypeSafeAI } from "../../src/providers.js" import { recordedTests } from "../recorded-test.js" const questions = { @@ -40,6 +40,14 @@ const zen = recordedTests({ metadata: { model: "jev-1.13-free" }, }) +const openrouter = recordedTests({ + prefix: "openrouter-evaluation", + provider: "openrouter", + protocol: "system-one", + requires: ["OPENROUTER_API_KEY"], + metadata: { model: "typesafe/jev-1.13" }, +}) + describe("experimental Evaluation recorded", () => { typesafe.effect("evaluates choice score and boolean questions", () => assertEvaluation( @@ -56,15 +64,24 @@ describe("experimental Evaluation recorded", () => { "opencode", ), ) + + openrouter.effect("evaluates choice score and boolean questions", () => + assertEvaluation( + OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" }).experimental.evaluation( + "typesafe/jev-1.13", + ), + "openrouter", + ), + ) }) -const assertEvaluation = ( - model: ReturnType, - metadataKey: "typesafe" | "opencode", +const assertEvaluation = ( + model: EvaluationModel, + metadataKey: "typesafe" | "opencode" | "openrouter", ) => Effect.gen(function* () { const response = yield* Evaluation.run({ model, state, questions }) - expect(response.model).toStartWith("jev-") + expect(response.model).toContain("jev-") expect(response.answers.department.type).toBe("choice") expect(response.answers.department.choice).toBe("billing") expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)