feat(ai): add gateway evaluation providers (#50665)

packages/ai/README.md

@@ -66,7 +66,17 @@ console.log(response.answers.refund.probability)
 ```
 
 `TypeSafeAI` reads `TYPESAFE_API_KEY`. `OpenCodeZen` exposes the same selector and reads
-`OPENCODE_API_KEY`. The common API uses `boolean`; System One routes lower it to native `noul`.
+`OPENCODE_API_KEY`. OpenRouter and Vercel AI Gateway use the same provider shape:
+
+```ts
+import { OpenRouter, VercelAIGateway } from "@opencode/ai/providers"
+
+OpenRouter.configure().experimental.evaluation("typesafe/jev-1.13")
+VercelAIGateway.configure().experimental.evaluation("typesafe-ai/jev")
+```
+
+OpenRouter reads `OPENROUTER_API_KEY`. Vercel reads `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN`.
+The common API uses `boolean`; System One routes lower it to native `noul`.
 Choice and score confidence plus score legends remain available in provider metadata, and the
 provider's rounded probabilities are returned unchanged.
 

packages/ai/src/experimental/evaluation-client.ts

@@ -1,6 +1,6 @@
 import { Context, Effect, Layer } from "effect"
 import { RequestExecutor } from "../route/executor.js"
-import { mergeHttpOptions, type AIError } from "../schema/index.js"
+import { AIError, InvalidProviderOutputError, mergeHttpOptions } from "../schema/index.js"
 import { sanitizeSurrogates } from "../utils/sanitize.js"
 import {
   type EvaluationOptions,
@@ -30,17 +30,59 @@ export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer
     const executor = yield* RequestExecutor.Service
     return Service.of({
       evaluate: (request) =>
-        request.model.route.evaluate(
-          {
-            ...sanitizeSurrogates({
-              ...request,
-              model: undefined,
-              http: mergeHttpOptions(request.model.http, request.http),
+        request.model.route
+          .evaluate(
+            {
+              ...sanitizeSurrogates({
+                ...request,
+                model: undefined,
+                http: mergeHttpOptions(request.model.http, request.http),
+              }),
+              model: request.model,
+            },
+            executor.execute,
+          )
+          .pipe(
+            Effect.flatMap((response) => {
+              const questions = Object.entries(request.questions)
+              if (
+                questions.length === Object.keys(response.answers).length &&
+                questions.every(([id, question]) => {
+                  const answer = response.answers[id]
+                  if (question.type === "boolean") return answer?.type === "boolean"
+                  if (question.type === "choice") {
+                    if (answer?.type !== "choice" || !Object.hasOwn(question.criteria, answer.choice)) return false
+                    if (answer.probabilities === undefined) return true
+                    const keys = Object.keys(question.criteria)
+                    const probabilities = answer.probabilities
+                    return (
+                      Object.keys(probabilities).length === keys.length &&
+                      keys.every((key) => Object.hasOwn(probabilities, key))
+                    )
+                  }
+                  if (answer?.type !== "score" || answer.score < 0 || answer.score > question.criteria.length - 1)
+                    return false
+                  if (answer.probabilities === undefined) return true
+                  const keys = question.criteria.map((_, index) => String(index))
+                  const probabilities = answer.probabilities
+                  return (
+                    Object.keys(probabilities).length === keys.length &&
+                    keys.every((key) => Object.hasOwn(probabilities, key))
+                  )
+                })
+              )
+                return Effect.succeed(response as EvaluationResponseFor<typeof request.questions>)
+              return Effect.fail(
+                new AIError({
+                  reason: new InvalidProviderOutputError({
+                    route: request.model.route.id,
+                    message: "Evaluation answers do not match the requested questions",
+                    cause: response.answers,
+                  }),
+                }),
+              )
             }),
-            model: request.model,
-          },
-          executor.execute,
-        ),
+          ),
     })
   }),
 )

packages/ai/src/experimental/evaluation.ts

@@ -105,10 +105,10 @@ export type EvaluationOptions = Record<string, unknown>
 
 export interface EvaluationRoute<Options extends EvaluationOptions = EvaluationOptions> {
   readonly id: string
-  readonly evaluate: <const Questions extends EvaluationQuestions>(
-    request: EvaluationRequestFor<Options, Questions>,
+  readonly evaluate: (
+    request: EvaluationRequestFor<Options>,
     execute: Execute,
-  ) => Effect.Effect<EvaluationResponseFor<Questions>, AIError>
+  ) => Effect.Effect<EvaluationResponse, AIError>
 }
 
 export class EvaluationModel<Options extends EvaluationOptions = EvaluationOptions> {

packages/ai/src/experimental/system-one.ts

@@ -1,17 +1,14 @@
 import { Effect, Schema } from "effect"
 import { Headers, HttpClientRequest } from "effect/unstable/http"
 import {
-  BooleanAnswer,
-  ChoiceAnswer,
   ChoiceQuestion,
   EvaluationInput,
   EvaluationModel,
   EvaluationResponse,
   EvaluationRounding,
-  ScoreAnswer,
   ScoreQuestion,
-  type EvaluationResponseFor,
-  type EvaluationRoute,
+  type EvaluationAnswer,
+  type EvaluationOptions,
 } from "./evaluation.js"
 import { Auth, type Definition as AuthDefinition } from "../route/auth.js"
 import {
@@ -72,15 +69,23 @@ const Score = Schema.Struct({
   legend: Schema.optional(Schema.Record(Schema.String, Schema.Json)),
   confidence: Schema.optional(Probability),
 })
-const NativeUsage = Schema.Struct({
-  input_tokens: Schema.optional(Schema.Number),
-  output_tokens: Schema.optional(Schema.Number),
+const Answer = Schema.Union([NoulAnswer, Choice, Score]).pipe(Schema.toTaggedUnion("type"))
+const NativeUsage = Schema.StructWithRest(
+  Schema.Struct({
+    input_tokens: Schema.optional(Schema.Number),
+    output_tokens: Schema.optional(Schema.Number),
+  }),
+  [Schema.Record(Schema.String, Schema.Unknown)],
+)
+const Response = Schema.Struct({
+  model: Schema.String,
+  answers: Schema.Record(Schema.String, Answer),
+  usage: Schema.optional(NativeUsage),
+  id: Schema.optional(Schema.String),
+  provider: Schema.optional(Schema.String),
+  provider_metadata: Schema.optional(Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown))),
 })
 
-const encode = Schema.encodeUnknownEffect(Schema.fromJsonString(Request))
-const exact = (x: Readonly<Record<string, unknown>>, keys: ReadonlyArray<string>) =>
-  Object.keys(x).length === keys.length && keys.every((key) => Object.hasOwn(x, key))
-
 export interface ModelInput {
   readonly id: string | ModelID
   readonly provider: string
@@ -91,132 +96,99 @@ export interface ModelInput {
   readonly http?: HttpOptions
 }
 
-export const model = (cfg: ModelInput) => {
-  const route: EvaluationRoute = {
-    id: "system-one",
-    evaluate: (req, send) =>
-      Effect.gen(function* () {
-        const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`)
-        Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
-        const body = yield* encode({
-          ...mergeJsonRecords(req.options, req.http?.body),
-          model: req.model.id,
-          state: req.state,
-          questions: Object.fromEntries(
-            Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]),
-          ),
-        }).pipe(
-          Effect.mapError(
-            (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
-          ),
-        )
-        const headers = yield* Auth.toEffect(cfg.auth)({
-          request: req,
-          method: "POST",
-          url: url.toString(),
-          body,
-          headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }),
-        })
-        const res = yield* send(
-          HttpClientRequest.post(url).pipe(
-            HttpClientRequest.setHeaders(headers),
-            HttpClientRequest.bodyText(body, "application/json"),
-          ),
-        )
-        const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
-        const fail = (message: string, cause: unknown, body?: string) =>
-          new AIError({ reason: new InvalidProviderOutputError({ route: route.id, message, body, http, cause }) })
-        const text = yield* res.text.pipe(
-          Effect.mapError((cause) => fail("Failed to read the System One response", cause)),
-        )
-        const entries = Object.entries(req.questions)
-        const output = Schema.Struct({
-          model: Schema.String,
-          answers: Schema.Struct(
-            Object.fromEntries(
-              entries.map(([id, question]) => {
-                if (question.type === "boolean") return [id, NoulAnswer]
-                if (question.type === "choice") {
-                  const keys = Object.keys(question.criteria)
-                  return [
-                    id,
-                    Choice.pipe(
-                      Schema.refine(
-                        (x): x is typeof x =>
-                          Object.hasOwn(question.criteria, x.choice) && exact(x.probabilities, keys),
-                        { message: `Question "${id}" returned an invalid choice answer` },
-                      ),
-                    ),
-                  ]
-                }
-                const keys = question.criteria.map((_, index) => String(index))
+export const model = <Options extends EvaluationOptions = EvaluationOptions>(cfg: ModelInput) =>
+  EvaluationModel.make<Options>({
+    id: cfg.id,
+    provider: cfg.provider,
+    http: cfg.http,
+    route: {
+      id: "system-one",
+      evaluate: (req, send) =>
+        Effect.gen(function* () {
+          const url = new URL(`${cfg.baseURL.replace(/\/$/, "")}/systemone`)
+          Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
+          const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({
+            ...mergeJsonRecords(req.options, req.http?.body),
+            model: req.model.id,
+            state: req.state,
+            questions: Object.fromEntries(
+              Object.entries(req.questions).map(([id, x]) => [id, x.type === "boolean" ? { ...x, type: "noul" } : x]),
+            ),
+          }).pipe(
+            Effect.mapError(
+              (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
+            ),
+          )
+          const headers = yield* Auth.toEffect(cfg.auth)({
+            request: req,
+            method: "POST",
+            url: url.toString(),
+            body,
+            headers: Headers.fromInput({ ...cfg.headers, ...req.http?.headers }),
+          })
+          const res = yield* send(
+            HttpClientRequest.post(url).pipe(
+              HttpClientRequest.setHeaders(headers),
+              HttpClientRequest.bodyText(body, "application/json"),
+            ),
+          )
+          const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
+          const fail = (message: string, cause: unknown, body?: string) =>
+            new AIError({ reason: new InvalidProviderOutputError({ route: "system-one", message, body, http, cause }) })
+          const text = yield* res.text.pipe(
+            Effect.mapError((cause) => fail("Failed to read the System One response", cause)),
+          )
+          const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(
+            Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)),
+          )
+
+          const confidence: Record<string, number> = {}
+          const legend: Record<string, Record<string, Schema.Json>> = {}
+          const answers = Object.fromEntries(
+            Object.entries(data.answers).map(([id, answer]): [string, EvaluationAnswer] => {
+              if (answer.type === "noul") return [id, { type: "boolean", probability: answer.noul }]
+              if (answer.type === "choice") {
+                if (answer.confidence !== undefined) confidence[id] = answer.confidence
                 return [
                   id,
-                  Score.pipe(
-                    Schema.refine(
-                      (x): x is typeof x =>
-                        x.score >= 0 && x.score <= question.criteria.length - 1 && exact(x.probabilities, keys),
-                      { message: `Question "${id}" returned an invalid score answer` },
-                    ),
-                  ),
+                  {
+                    type: "choice",
+                    choice: answer.choice,
+                    probabilities: answer.probabilities,
+                  },
                 ]
-              }),
-            ) as Record<string, Schema.Codec<unknown>>,
-          ),
-          usage: Schema.optional(NativeUsage),
-        })
-        const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(output))(text).pipe(
-          Effect.mapError((cause) => fail("System One returned an invalid response", cause, text)),
-        )
-
-        const confidence: Record<string, number> = {}
-        const legend: Record<string, Record<string, Schema.Json>> = {}
-        const answers = Object.fromEntries(
-          entries.map(([id, question]) => {
-            const answer = data.answers[id]
-            if (question.type === "boolean")
-              return [
-                id,
-                { type: "boolean", probability: (answer as typeof NoulAnswer.Type).noul } satisfies BooleanAnswer,
-              ]
-            if (question.type === "choice") {
-              const value = answer as typeof Choice.Type
-              if (value.confidence !== undefined) confidence[id] = value.confidence
-              return [
-                id,
-                { type: "choice", choice: value.choice, probabilities: value.probabilities } satisfies ChoiceAnswer,
-              ]
-            }
-            const value = answer as typeof Score.Type
-            if (value.confidence !== undefined) confidence[id] = value.confidence
-            if (value.legend !== undefined) legend[id] = value.legend
-            return [id, { type: "score", score: value.score, probabilities: value.probabilities } satisfies ScoreAnswer]
-          }),
-        )
-        const meta = {
-          ...(Object.keys(confidence).length === 0 ? {} : { confidence }),
-          ...(Object.keys(legend).length === 0 ? {} : { legend }),
-        }
-        return new EvaluationResponse({
-          model: ModelID.make(data.model),
-          answers,
-          usage: data.usage
-            ? new Usage({
-                inputTokens: data.usage.input_tokens,
-                outputTokens: data.usage.output_tokens,
-                totalTokens:
-                  data.usage.input_tokens === undefined && data.usage.output_tokens === undefined
-                    ? undefined
-                    : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0),
-                providerMetadata: { [cfg.providerMetadataKey]: data.usage },
-              })
-            : undefined,
-          rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
-          providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta },
-        }) as EvaluationResponseFor<typeof req.questions>
-      }),
-  }
-  return EvaluationModel.make({ id: cfg.id, provider: cfg.provider, route, http: cfg.http })
-}
+              }
+              if (answer.confidence !== undefined) confidence[id] = answer.confidence
+              if (answer.legend !== undefined) legend[id] = answer.legend
+              return [id, { type: "score", score: answer.score, probabilities: answer.probabilities }]
+            }),
+          )
+          const meta = {
+            ...(data.id === undefined ? {} : { responseId: data.id }),
+            ...(data.provider === undefined ? {} : { provider: data.provider }),
+            ...data.provider_metadata?.[cfg.providerMetadataKey],
+            ...(Object.keys(confidence).length === 0 ? {} : { confidence }),
+            ...(Object.keys(legend).length === 0 ? {} : { legend }),
+          }
+          return new EvaluationResponse({
+            model: ModelID.make(data.model),
+            answers,
+            usage: data.usage
+              ? new Usage({
+                  inputTokens: data.usage.input_tokens,
+                  outputTokens: data.usage.output_tokens,
+                  totalTokens:
+                    data.usage.input_tokens === undefined && data.usage.output_tokens === undefined
+                      ? undefined
+                      : (data.usage.input_tokens ?? 0) + (data.usage.output_tokens ?? 0),
+                  providerMetadata: { [cfg.providerMetadataKey]: data.usage },
+                })
+              : undefined,
+            rounding: new EvaluationRounding({ probabilityDecimals: 2, scoreDecimals: 2 }),
+            providerMetadata: Object.keys(meta).length === 0 ? undefined : { [cfg.providerMetadataKey]: meta },
+          })
+        }),
+    },
+  })
 
 export const SystemOne = { model } as const

packages/ai/src/providers/index.ts

@@ -28,6 +28,7 @@ export * as OpenCodeZen from "./opencode-zen.js"
 export * as OpenRouter from "./openrouter.js"
 export * as TogetherAI from "./togetherai.js"
 export * as TypeSafeAI from "./typesafe-ai.js"
+export * as VercelAIGateway from "./vercel-ai-gateway.js"
 export * as XAI from "./xai.js"
 export * as ZAI from "./zai.js"
 export * as ZAICodingPlan from "./zai-coding-plan.js"

packages/ai/src/providers/openrouter.ts

@@ -3,8 +3,9 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
 import { Endpoint } from "../route/endpoint.js"
 import { Protocol } from "../route/protocol.js"
 import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
-import { ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
+import { HttpOptions, ProviderID, type CacheHint, type ModelID } from "../schema/index.js"
 import type { ProviderPackage } from "../provider-package.js"
+import { SystemOne } from "../experimental/system-one.js"
 import { OpenAIChat } from "../protocols/openai-chat.js"
 import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
 import { isRecord } from "../protocols/shared.js"
@@ -71,6 +72,14 @@ export interface OpenRouterOptions {
 
 export type OpenRouterProviderOptionsInput = OpenRouterOptions
 
+export interface OpenRouterEvaluationOptions {
+  readonly [key: string]: unknown
+  readonly provider?: OpenRouterProviderRouting
+  readonly session_id?: string
+  readonly trace?: Readonly<Record<string, unknown>>
+  readonly user?: string
+}
+
 export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
   ProviderAuthOption<"optional"> & {
     readonly baseURL?: string
@@ -181,15 +190,27 @@ const configuredRoute = (input: LanguageModelOptions) => {
 
 export const configure = (input: LanguageModelOptions = {}) => {
   const route = configuredRoute(input)
+  const evaluation = (modelID: string | ModelID) =>
+    SystemOne.model<OpenRouterEvaluationOptions>({
+      id: modelID,
+      provider: id,
+      providerMetadataKey: "openrouter",
+      auth: AuthOptions.bearer(input, "OPENROUTER_API_KEY"),
+      baseURL: input.baseURL ?? baseURL,
+      headers: input.headers,
+      http: input.http === undefined ? undefined : HttpOptions.make(input.http),
+    })
   return {
     id,
     model: (modelID: string | ModelID) =>
       route.model<OpenRouterProviderOptionsInput>({ id: modelID, compatibility: { supportsPromptCacheKey: true } }),
+    experimental: { evaluation },
     configure,
   }
 }
 
 export const provider = configure()
+export const experimental = provider.experimental
 export const model: ProviderPackage.Definition<Settings, OpenRouterProviderOptionsInput>["model"] = (
   modelID,
   { apiKey, baseURL, body, headers, ...providerOptions },

packages/ai/src/providers/vercel-ai-gateway.ts

@@ -0,0 +1,148 @@
+import { Effect, Schema } from "effect"
+import { Headers, HttpClientRequest } from "effect/unstable/http"
+import {
+  EvaluationAnswer,
+  EvaluationInput,
+  EvaluationModel,
+  EvaluationQuestion,
+  EvaluationResponse,
+  EvaluationRounding,
+} from "../experimental/evaluation.js"
+import { Auth } from "../route/auth.js"
+import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
+import {
+  AIError,
+  HttpContext,
+  HttpOptions,
+  InvalidProviderOutputError,
+  InvalidRequestError,
+  ModelID,
+  ProviderID,
+  ProviderMetadata,
+  Usage,
+} from "../schema/index.js"
+
+export const id = ProviderID.make("vercel-ai-gateway")
+const baseURL = "https://ai-gateway.vercel.sh/v1"
+
+export interface EvaluationOptions {
+  readonly [key: string]: unknown
+  readonly gateway?: Readonly<{
+    readonly [key: string]: unknown
+    readonly zeroDataRetention?: boolean
+    readonly only?: ReadonlyArray<string>
+  }>
+}
+
+export type Options = ProviderAuthOption<"optional"> & {
+  readonly baseURL?: string
+  readonly headers?: Record<string, string>
+  readonly http?: HttpOptions.Input
+}
+
+const Request = Schema.StructWithRest(
+  Schema.Struct({
+    model: Schema.String,
+    state: EvaluationInput,
+    questions: Schema.Record(Schema.String, EvaluationQuestion),
+    providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  }),
+  [Schema.Record(Schema.String, Schema.Any)],
+)
+const Response = Schema.Struct({
+  model: Schema.optional(Schema.String),
+  answers: Schema.Record(Schema.String, EvaluationAnswer),
+  usage: Schema.optional(
+    Schema.Struct({
+      inputTokens: Schema.optional(Schema.Number),
+      outputTokens: Schema.optional(Schema.Number),
+    }),
+  ),
+  rounding: Schema.optional(EvaluationRounding),
+  providerMetadata: Schema.optional(ProviderMetadata),
+})
+
+export const configure = (input: Options = {}) => {
+  const evaluation = (modelID: string | ModelID) =>
+    EvaluationModel.make<EvaluationOptions>({
+      id: modelID,
+      provider: id,
+      http: input.http === undefined ? undefined : HttpOptions.make(input.http),
+      route: {
+        id: "vercel-evaluation",
+        evaluate: (req, send) =>
+          Effect.gen(function* () {
+            const url = new URL(`${(input.baseURL ?? baseURL).replace(/\/$/, "")}/evaluate`)
+            Object.entries(req.http?.query ?? {}).forEach(([key, value]) => url.searchParams.set(key, value))
+            const body = yield* Schema.encodeUnknownEffect(Schema.fromJsonString(Request))({
+              ...req.http?.body,
+              model: req.model.id,
+              state: req.state,
+              questions: req.questions,
+              providerOptions: req.options,
+            }).pipe(
+              Effect.mapError(
+                (cause) => new AIError({ reason: new InvalidRequestError({ message: cause.message, cause }) }),
+              ),
+            )
+            const headers = yield* Auth.toEffect(
+              AuthOptions.bearer(input, ["AI_GATEWAY_API_KEY", "VERCEL_OIDC_TOKEN"]),
+            )({
+              request: req,
+              method: "POST",
+              url: url.toString(),
+              body,
+              headers: Headers.fromInput({ ...input.headers, ...req.http?.headers }),
+            })
+            const res = yield* send(
+              HttpClientRequest.post(url).pipe(
+                HttpClientRequest.setHeaders(headers),
+                HttpClientRequest.bodyText(body, "application/json"),
+              ),
+            )
+            const http = new HttpContext({ url: res.request.url, status: res.status, headers: res.headers })
+            const fail = (message: string, cause: unknown, body?: string) =>
+              new AIError({
+                reason: new InvalidProviderOutputError({
+                  route: "vercel-evaluation",
+                  message,
+                  body,
+                  http,
+                  cause,
+                }),
+              })
+            const text = yield* res.text.pipe(
+              Effect.mapError((cause) => fail("Failed to read the Vercel AI Gateway evaluation response", cause)),
+            )
+            const data = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Response))(text).pipe(
+              Effect.mapError((cause) =>
+                fail("Vercel AI Gateway returned an invalid evaluation response", cause, text),
+              ),
+            )
+            return new EvaluationResponse({
+              model: ModelID.make(data.model ?? req.model.id),
+              answers: data.answers,
+              usage: data.usage
+                ? new Usage({
+                    inputTokens: data.usage.inputTokens,
+                    outputTokens: data.usage.outputTokens,
+                    totalTokens:
+                      data.usage.inputTokens === undefined && data.usage.outputTokens === undefined
+                        ? undefined
+                        : (data.usage.inputTokens ?? 0) + (data.usage.outputTokens ?? 0),
+                    providerMetadata: { gateway: data.usage },
+                  })
+                : undefined,
+              rounding: data.rounding,
+              providerMetadata: data.providerMetadata,
+            })
+          }),
+      },
+    })
+  return { id, experimental: { evaluation }, configure }
+}
+
+export const provider = configure()
+export const experimental = provider.experimental
+
+export * as VercelAIGateway from "./vercel-ai-gateway.js"

packages/ai/test/evaluation.test.ts

@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
 import { Effect, Layer } from "effect"
 import { HttpClientRequest } from "effect/unstable/http"
 import { Evaluation, EvaluationClient } from "../src/experimental.js"
-import { OpenCodeZen, TypeSafeAI } from "../src/providers.js"
+import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
 import { it } from "./lib/effect.js"
 import { dynamicResponse } from "./lib/http.js"
 
@@ -154,6 +154,135 @@ describe("experimental Evaluation", () => {
     ),
   )
 
+  it.effect("evaluates through OpenRouter System One", () =>
+    Evaluation.run({
+      model: OpenRouter.configure({
+        apiKey: "openrouter-key",
+        baseURL: "https://openrouter.test/api/v1",
+      }).experimental.evaluation("typesafe/jev-1.13"),
+      state: "refund",
+      questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
+      options: { user: "user-1", session_id: "session-1" },
+    }).pipe(
+      Effect.tap((response) =>
+        Effect.sync(() => {
+          expect(response.answers.refund.probability).toBe(0.98)
+          expect(response.providerMetadata?.openrouter).toMatchObject({ responseId: "gen-1", provider: "TypeSafe" })
+          expect(response.usage?.providerMetadata?.openrouter).toEqual({
+            input_tokens: 10,
+            output_tokens: 2,
+            cost: 0.0001,
+          })
+        }),
+      ),
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(
+            dynamicResponse((input) => {
+              expect(input.request.url).toBe("https://openrouter.test/api/v1/systemone")
+              expect(input.request.headers.authorization).toBe("Bearer openrouter-key")
+              expect(JSON.parse(input.text)).toMatchObject({
+                model: "typesafe/jev-1.13",
+                user: "user-1",
+                session_id: "session-1",
+                questions: { refund: { type: "noul" } },
+              })
+              return Effect.succeed(
+                input.respond(
+                  JSON.stringify({
+                    id: "gen-1",
+                    model: "typesafe/jev-1.13-20260917",
+                    provider: "TypeSafe",
+                    answers: { refund: { type: "noul", noul: 0.98 } },
+                    usage: { input_tokens: 10, output_tokens: 2, cost: 0.0001 },
+                  }),
+                  { headers: { "content-type": "application/json" } },
+                ),
+              )
+            }),
+          ),
+        ),
+      ),
+    ),
+  )
+
+  it.effect("evaluates through Vercel AI Gateway", () =>
+    Evaluation.run({
+      model: VercelAIGateway.configure({
+        apiKey: "gateway-key",
+        baseURL: "https://gateway.test/v1/",
+      }).experimental.evaluation("typesafe-ai/jev"),
+      state: "refund",
+      questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
+      options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
+    }).pipe(
+      Effect.tap((response) =>
+        Effect.sync(() => {
+          expect(response.answers.refund.probability).toBe(0.98)
+          expect(response.usage?.totalTokens).toBe(12)
+          expect(response.providerMetadata?.gateway).toMatchObject({ generationId: "gen-1", cost: "0.0001" })
+        }),
+      ),
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(
+            dynamicResponse((input) => {
+              expect(input.request.url).toBe("https://gateway.test/v1/evaluate")
+              expect(input.request.headers.authorization).toBe("Bearer gateway-key")
+              expect(JSON.parse(input.text)).toEqual({
+                model: "typesafe-ai/jev",
+                state: "refund",
+                questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
+                providerOptions: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
+              })
+              return Effect.succeed(
+                input.respond(
+                  JSON.stringify({
+                    model: "typesafe-ai/jev",
+                    answers: { refund: { type: "boolean", probability: 0.98 } },
+                    usage: { inputTokens: 10, outputTokens: 2 },
+                    providerMetadata: { gateway: { generationId: "gen-1", cost: "0.0001" } },
+                  }),
+                  { headers: { "content-type": "application/json" } },
+                ),
+              )
+            }),
+          ),
+        ),
+      ),
+    ),
+  )
+
+  it.effect("rejects answers that do not match their questions", () =>
+    Evaluation.run({
+      model: VercelAIGateway.configure({
+        apiKey: "gateway-key",
+        baseURL: "https://gateway.test/v1",
+      }).experimental.evaluation("typesafe-ai/jev"),
+      state: "refund",
+      questions: { refund: { type: "boolean", instructions: "Is a refund requested?" } },
+    }).pipe(
+      Effect.flip,
+      Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidProviderOutput"))),
+      Effect.provide(
+        EvaluationClient.layer.pipe(
+          Layer.provide(
+            dynamicResponse((input) =>
+              Effect.succeed(
+                input.respond(
+                  JSON.stringify({
+                    answers: { refund: { type: "choice", choice: "yes", probabilities: { yes: 1 } } },
+                  }),
+                  { headers: { "content-type": "application/json" } },
+                ),
+              ),
+            ),
+          ),
+        ),
+      ),
+    ),
+  )
+
   it.effect("rejects malformed questions before network I/O", () =>
     Effect.gen(function* () {
       const error = yield* Evaluation.run({

packages/ai/test/evaluation.types.ts

@@ -1,7 +1,7 @@
 import { Effect } from "effect"
 import { Evaluation, EvaluationClient, EvaluationModel, type EvaluationRoute } from "../src/experimental.js"
 import type { Service } from "../src/experimental/evaluation-client.js"
-import { OpenCodeZen, TypeSafeAI } from "../src/providers.js"
+import { OpenCodeZen, OpenRouter, TypeSafeAI, VercelAIGateway } from "../src/providers.js"
 
 type Requirements<T> = T extends Effect.Effect<infer _A, infer _E, infer R> ? R : never
 type Success<T> = T extends Effect.Effect<infer A, infer _E, infer _R> ? A : never
@@ -58,3 +58,31 @@ Evaluation.run({
   questions: { ok: { type: "boolean", instructions: "OK?" } },
   options: { temperature: "high" },
 })
+
+Evaluation.run({
+  model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"),
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { provider: { zdr: true }, session_id: "session-1", user: "user-1" },
+})
+// @ts-expect-error OpenRouter session IDs are strings.
+Evaluation.run({
+  model: OpenRouter.experimental.evaluation("typesafe/jev-1.13"),
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { session_id: 1 },
+})
+
+Evaluation.run({
+  model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"),
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { gateway: { zeroDataRetention: true, only: ["typesafe-ai"] } },
+})
+// @ts-expect-error Vercel zero-data-retention controls are boolean.
+Evaluation.run({
+  model: VercelAIGateway.experimental.evaluation("typesafe-ai/jev"),
+  state: "hello",
+  questions: { ok: { type: "boolean", instructions: "OK?" } },
+  options: { gateway: { zeroDataRetention: "yes" } },
+})

packages/ai/test/exports.test.ts

@@ -13,6 +13,7 @@ import {
   OpenAICompatible,
   OpenRouter,
   TypeSafeAI,
+  VercelAIGateway,
   XAI,
 } from "@opencode/ai/providers"
 import {
@@ -72,8 +73,10 @@ describe("public exports", () => {
     expect(CloudflareWorkersAI.configure).toBeFunction()
     expect(CloudflareWorkersAI.configure({ accountId: "fixture", apiKey: "fixture" }).model).toBeFunction()
     expect(OpenRouter.model).toBeFunction()
+    expect(OpenRouter.experimental.evaluation).toBeFunction()
     expect(TypeSafeAI.experimental.evaluation).toBeFunction()
     expect(OpenCodeZen.experimental.evaluation).toBeFunction()
+    expect(VercelAIGateway.experimental.evaluation).toBeFunction()
     expect(XAI.model).toBeFunction()
     expect(XAI.provider.responses).toBe(XAI.responses)
     expect(XAI.provider.chat).toBe(XAI.chat)

packages/ai/test/fixtures/recordings/openrouter-evaluation/evaluates-choice-score-and-boolean-questions.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "metadata": {
+    "model": "typesafe/jev-1.13",
+    "tags": [
+      "prefix:openrouter-evaluation",
+      "provider:openrouter",
+      "protocol:system-one"
+    ],
+    "name": "openrouter-evaluation/evaluates-choice-score-and-boolean-questions",
+    "recordedAt": "2026-09-22T14:54:46.656Z"
+  },
+  "interactions": [
+    {
+      "transport": "http",
+      "request": {
+        "method": "POST",
+        "url": "https://openrouter.ai/api/v1/systemone",
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"typesafe/jev-1.13\",\"state\":\"I was charged twice for the same invoice. Please refund the duplicate payment today.\",\"questions\":{\"department\":{\"type\":\"choice\",\"instructions\":\"Which team should handle this support request?\",\"criteria\":{\"billing\":\"Payments, invoices, refunds, or failed charges\",\"technical\":\"Bugs, outages, or integrations\",\"sales\":\"Pricing, upgrades, or new accounts\"}},\"urgency\":{\"type\":\"score\",\"instructions\":\"How urgent is this support request?\",\"criteria\":[\"Can wait for normal support\",\"Needs prompt attention\",\"Actively blocking revenue\"]},\"refund\":{\"type\":\"noul\",\"instructions\":\"Is the customer asking for a refund?\"}}}"
+      },
+      "response": {
+        "status": 200,
+        "headers": {
+          "content-type": "application/json"
+        },
+        "body": "{\"model\":\"typesafe/jev-1.13-20260917\",\"answers\":{\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"probabilities\":{\"billing\":1,\"sales\":0,\"technical\":0},\"confidence\":1},\"urgency\":{\"type\":\"score\",\"score\":1.02,\"legend\":{\"0\":\"Can wait for normal support\",\"1\":\"Needs prompt attention\",\"2\":\"Actively blocking revenue\"},\"probabilities\":{\"0\":0.01,\"1\":0.96,\"2\":0.03},\"confidence\":0.95},\"refund\":{\"type\":\"noul\",\"noul\":0.99}},\"usage\":{\"input_tokens\":422,\"output_tokens\":69,\"cost\":0.000017724},\"id\":\"gen-dec-1790088886-JRlffOLUwQdx4Lltu2se\",\"provider\":\"TypeSafe\"}"
+      }
+    }
+  ]
+}

packages/ai/test/provider/evaluation.recorded.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect } from "bun:test"
 import { Effect } from "effect"
-import { Evaluation } from "../../src/experimental.js"
-import { OpenCodeZen, TypeSafeAI } from "../../src/providers.js"
+import { Evaluation, EvaluationModel, type EvaluationOptions } from "../../src/experimental.js"
+import { OpenCodeZen, OpenRouter, TypeSafeAI } from "../../src/providers.js"
 import { recordedTests } from "../recorded-test.js"
 
 const questions = {
@@ -40,6 +40,14 @@ const zen = recordedTests({
   metadata: { model: "jev-1.13-free" },
 })
 
+const openrouter = recordedTests({
+  prefix: "openrouter-evaluation",
+  provider: "openrouter",
+  protocol: "system-one",
+  requires: ["OPENROUTER_API_KEY"],
+  metadata: { model: "typesafe/jev-1.13" },
+})
+
 describe("experimental Evaluation recorded", () => {
   typesafe.effect("evaluates choice score and boolean questions", () =>
     assertEvaluation(
@@ -56,15 +64,24 @@ describe("experimental Evaluation recorded", () => {
       "opencode",
     ),
   )
+
+  openrouter.effect("evaluates choice score and boolean questions", () =>
+    assertEvaluation(
+      OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" }).experimental.evaluation(
+        "typesafe/jev-1.13",
+      ),
+      "openrouter",
+    ),
+  )
 })
 
-const assertEvaluation = (
-  model: ReturnType<typeof TypeSafeAI.experimental.evaluation>,
-  metadataKey: "typesafe" | "opencode",
+const assertEvaluation = <Options extends EvaluationOptions>(
+  model: EvaluationModel<Options>,
+  metadataKey: "typesafe" | "opencode" | "openrouter",
 ) =>
   Effect.gen(function* () {
     const response = yield* Evaluation.run({ model, state, questions })
-    expect(response.model).toStartWith("jev-")
+    expect(response.model).toContain("jev-")
     expect(response.answers.department.type).toBe("choice")
     expect(response.answers.department.choice).toBe("billing")
     expect(response.answers.department.probabilities?.billing).toBeGreaterThan(0.5)