PHP code example of aysnc / llm-eval

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

aysnc / llm-eval example snippets

use Aysnc\AI\LlmEval\Providers\AnthropicProvider;

return [
    'provider'    => new AnthropicProvider(getenv('ANTHROPIC_API_KEY')),
    'directory'   => __DIR__ . '/evals',
    'cache'       => true,
    'cacheTtl'    => 0,
    'parallel'    => false,
    'concurrency' => 5,
];

// evals/simple.php


use Aysnc\AI\LlmEval\Dataset\Dataset;
use Aysnc\AI\LlmEval\LlmEval;

$dataset = Dataset::fromArray([
    ['prompt' => 'What is 2+2? Reply with just the number.', 'expected' => '4'],
    ['prompt' => 'What is the capital of France? Reply with just the city name.', 'expected' => 'Paris'],
    ['prompt' => 'Is the sky blue? Reply with just yes or no.', 'expected' => 'yes'],
]);

return LlmEval::create('quick-start')
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->contains($testCase->getExpected(), caseSensitive: false);
    });

// Inline array
$dataset = Dataset::fromArray([
    ['prompt' => 'What is 2+2?', 'expected' => '4'],
]);

// CSV file (columns: prompt, expected)
$dataset = Dataset::fromCsv(__DIR__ . '/data/capitals.csv');

// JSON file (array of objects with prompt + expected keys)
$dataset = Dataset::fromJson(__DIR__ . '/data/questions.json');

// Single — accessed via $testCase->getExpected()
['prompt' => 'What is 2+2?', 'expected' => '4']

// Multiple — accessed via $testCase->getExpected('name'), $testCase->getExpected('age')
['prompt' => 'Return JSON with name and age.', 'expected' => ['name' => 'Alice', 'age' => '30']]

$expect->contains('Paris');
$expect->contains('paris', caseSensitive: false);
$expect->notContains('London');
$expect->matchesRegex('/\d{4}-\d{2}-\d{2}/');
$expect->minLength(10);
$expect->maxLength(500);

$expect->isJson();

$expect->assert(new MyCustomAssertion());

// evals/json-output.php
$dataset = Dataset::fromArray([
    [
        'prompt' => 'Return a JSON object with keys "name" and "age". Use name "Alice" and age 30. Only output JSON.',
        'expected' => ['name' => 'Alice', 'age' => '30'],
    ],
    [
        'prompt' => 'Return a JSON array of three colors: red, green, blue. Only output JSON.',
        'expected' => 'red',
    ],
]);

return LlmEval::create('json-output')
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->isJson()
            ->contains($testCase->getExpected())
            ->contains($testCase->getExpected('name'));
    });

// evals/tool-test.php
$tools = [
    [
        'name' => 'get_weather',
        'description' => 'Get weather for a location',
        'input_schema' => [
            'type' => 'object',
            'properties' => [
                'location' => ['type' => 'string'],
            ],
            '

$expect->calledTool('get_weather');
$expect->calledTool('get_weather', times: 2);
$expect->toolCallHasParam('get_weather', 'location');
$expect->toolCallHasParam('get_weather', 'location', 'Paris');
$expect->calledToolCount(3);
$expect->didNotCallTool('dangerous_function');

// evals/math-agent.php
use Aysnc\AI\LlmEval\Dataset\Dataset;
use Aysnc\AI\LlmEval\LlmEval;
use Aysnc\AI\LlmEval\Providers\CallableToolExecutor;
use Aysnc\AI\LlmEval\Providers\ToolCall;
use Aysnc\AI\LlmEval\Providers\ToolResult;

$tools = [
    [
        'name' => 'calculate',
        'description' => 'Evaluate a math expression',
        'input_schema' => [
            'type' => 'object',
            'properties' => [
                'expression' => ['type' => 'string'],
            ],
            't')
    ->withTools($tools)
    ->executor($executor)
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->contains($testCase->getExpected())
            ->usedTool('calculate')
            ->turnCount(2);
    });

$expect->turnCount(2);
$expect->usedTool('calculate');
$expect->conversationContains('42');

$dataset = Dataset::fromArray([
    [
        'turns' => [
            ['prompt' => 'What is the weather in Paris?', 'expected' => '22'],
            ['prompt' => 'Now check Tokyo', 'expected' => '18'],
            ['prompt' => 'Which city was warmer?', 'expected' => 'Paris'],
        ],
    ],
]);

return LlmEval::createConversation('multi-turn')
    ->withTools($tools)
    ->executor($executor)
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->contains($testCase->getExpected());

        if ($testCase->getTurn() <= 2) {
            $expect->usedTool('get_weather');
        }
    });

// evals/quality-check.php
use Aysnc\AI\LlmEval\Providers\AnthropicProvider;

$judge = new AnthropicProvider(getenv('ANTHROPIC_API_KEY'));

return LlmEval::create('quality-check')
    ->dataset($dataset)
    ->assertions(function ($expect) use ($judge): void {
        $expect->judgedBy(
            judge: $judge,
            criteria: 'Is this response helpful, accurate, and concise?',
            threshold: 0.8,
        );
    });

return LlmEval::createConversation('multi-turn')
    ->withTools($tools)
    ->executor($executor)
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->contains($testCase->getExpected());
    })
    ->judge($judge, 'Did the model correctly identify the warmer city based on the earlier temperatures?');

$provider = new AnthropicProvider(
    apiKey: getenv('ANTHROPIC_API_KEY'),
);

use Aysnc\AI\LlmEval\Providers\BedrockProvider;

// Explicit credentials
$provider = new BedrockProvider(
    region: 'us-east-1',
    accessKeyId: 'AKIA...',
    secretAccessKey: 'secret...',
);

// Or default credential chain (env vars, ~/.aws/credentials, IAM role)
$provider = new BedrockProvider(region: 'us-east-1');

return LlmEval::create('eval-name')
    ->model('claude-opus-4-20250514')
    ->dataset($dataset)
    ->assertions($assertions);

$provider = new AnthropicProvider(getenv('ANTHROPIC_API_KEY'));

$results = LlmEval::create('quick-start')
    ->provider($provider)
    ->dataset($dataset)
    ->assertions(function ($expect, $testCase): void {
        $expect->contains($testCase->getExpected());
    })
    ->runAll();

echo "Pass rate: {$results->passRatePercent()}\n";
// Pass rate: 100.0%