Manual evaluation API

import { Laminar, LaminarClient, observe } from '@lmnr-ai/lmnr';
import { OpenAI } from 'openai';

Laminar.initialize({
  projectApiKey: process.env.LMNR_PROJECT_API_KEY,
  instrumentModules: { OpenAI },
});

const client = new LaminarClient({
  projectApiKey: process.env.LMNR_PROJECT_API_KEY,
});

const openai = new OpenAI();

import os
from lmnr import Laminar, LaminarClient, observe
from openai import AsyncOpenAI

Laminar.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])

client = LaminarClient(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
openai_client = AsyncOpenAI()

const runExecutor = async (testCase: { data: { country: string }; target: string }) =>
  observe(
    { name: 'executor', spanType: 'EXECUTOR', input: testCase.data },
    async () => {
      const response = await openai.chat.completions.create({
        model: 'gpt-5-mini',
        messages: [
          {
            role: 'user',
            content:
              `What is the capital of ${testCase.data.country}? ` +
              'Answer only with the capital, no other text.',
          },
        ],
      });
      return response.choices[0].message.content ?? '';
    },
  );

const accuracy = async (output: string, target: string) =>
  observe(
    { name: 'accuracy', spanType: 'EVALUATOR', input: { output, target } },
    async () => (output.toLowerCase().includes(target.toLowerCase()) ? 1 : 0),
  );

const lengthOk = async (output: string) =>
  observe(
    { name: 'length_ok', spanType: 'EVALUATOR', input: { output } },
    async () => (output.length > 0 && output.length < 50 ? 1 : 0),
  );

@observe(name="executor", span_type="EXECUTOR")
async def run_executor(test_case):
    response = await openai_client.chat.completions.create(
        model="gpt-5-mini",
        messages=[
            {
                "role": "user",
                "content": (
                    f"What is the capital of {test_case['data']['country']}? "
                    "Answer only with the capital, no other text."
                ),
            }
        ],
    )
    return response.choices[0].message.content or ""


@observe(name="accuracy", span_type="EVALUATOR")
async def accuracy(output, target):
    return 1 if target.lower() in output.lower() else 0


@observe(name="length_ok", span_type="EVALUATOR")
async def length_ok(output, target=None):
    return 1 if 0 < len(output) < 50 else 0

const testData = [
  { data: { country: 'France' }, target: 'Paris' },
  { data: { country: 'Germany' }, target: 'Berlin' },
  { data: { country: 'Japan' }, target: 'Tokyo' },
];

const evalId = await client.evals.createEvaluation(
  'Manual capitals eval',
  'capitals-manual',
  { model: 'gpt-5-mini', run_by: 'manual-api-demo' },
);

for (let i = 0; i < testData.length; i++) {
  const testCase = testData[i];

  await observe(
    { name: 'evaluation', spanType: 'EVALUATION', input: { testCase } },
    async () => {
      // Pre-register the datapoint with the current trace ID.
      // The row shows up in the UI now, before the executor runs.
      const datapointId = await client.evals.createDatapoint({
        evalId,
        data: testCase.data,
        target: testCase.target,
        index: i,
        traceId: Laminar.getTraceId(),
      });

      const output = await runExecutor(testCase);
      const accuracyScore = await accuracy(output, testCase.target);
      const lengthOkScore = await lengthOk(output);

      await client.evals.updateDatapoint({
        evalId,
        datapointId,
        scores: { accuracy: accuracyScore, length_ok: lengthOkScore },
        executorOutput: { response: output, model: 'gpt-5-mini' },
      });
    },
  );
}

await Laminar.flush();

test_data = [
    {"data": {"country": "France"}, "target": "Paris"},
    {"data": {"country": "Germany"}, "target": "Berlin"},
    {"data": {"country": "Japan"}, "target": "Tokyo"},
]


async def run_one(eval_id, index, test_case):
    # Phase 2: pre-register the datapoint. A row appears in the UI now.
    datapoint_id = client.evals.create_datapoint(
        eval_id=eval_id,
        data=test_case["data"],
        target=test_case["target"],
        index=index,
    )

    @observe(name="evaluation", span_type="EVALUATION")
    async def run_inside_span():
        # Link the datapoint to the evaluation trace that is now active.
        trace_id = Laminar.get_trace_id()
        client.evals.update_datapoint(
            eval_id=eval_id,
            datapoint_id=datapoint_id,
            scores={},
            trace_id=trace_id,
        )

        output = await run_executor(test_case)
        accuracy_score = await accuracy(output, test_case["target"])
        length_ok_score = await length_ok(output, test_case["target"])
        return output, {"accuracy": accuracy_score, "length_ok": length_ok_score}

    output, scores = await run_inside_span()

    # Phase 3: write the final executor output and scores.
    client.evals.update_datapoint(
        eval_id=eval_id,
        datapoint_id=datapoint_id,
        executor_output={"response": output, "model": "gpt-5-mini"},
        scores=scores,
    )


async def main():
    eval_id = client.evals.create_evaluation(
        name="Manual capitals eval",
        group_name="capitals-manual",
        metadata={"model": "gpt-5-mini", "run_by": "manual-api-demo"},
    )

    for index, test_case in enumerate(test_data):
        await run_one(eval_id, index, test_case)

    Laminar.flush()

await client.evals.update({
  evalId,
  name: 'Manual capitals eval (complete)',
  metadata: { model: 'gpt-5-mini', run_by: 'manual-api-demo', status: 'complete' },
});

client.evals.update_evaluation(
    eval_id=eval_id,
    name="Manual capitals eval (complete)",
    metadata={"model": "gpt-5-mini", "run_by": "manual-api-demo", "status": "complete"},
)