PHP code example of casawatt / laravel-ai-agent-evaluation

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

casawatt / laravel-ai-agent-evaluation example snippets

use Laravel\Ai\Enums\Lab;
use Casawatt\LaravelAiAgentEvaluation\Attributes\EvaluationCase;
use Casawatt\LaravelAiAgentEvaluation\Evaluation;

return new class extends Evaluation
{
    protected string $agent = \App\Ai\Agents\SalesCoach::class;

    public function setUp(): void
    {
        $this->variant(Lab::Mistral, 'mistral-small-3.2-24b-instruct');
        $this->variant(Lab::OpenRouter, 'google/gemma-3-27b-it');
        $this->variant(Lab::OpenRouter, 'openai/gpt-oss-120b');
        $this->variant('scaleway', 'gpt-oss-120b');
    }

    #[EvaluationCase]
    public function contains_hello(): void
    {
        $this->agent(prompt: 'Say hello to the user')
            ->assertContains('hello');
    }

    #[EvaluationCase(description: 'Handles file attachments')]
    public function with_attachments(): void
    {
        $this->agent(
            prompt: 'Summarize this document',
            attachments: [
                'path/to/document.pdf',
                'https://example.com/data.csv',
            ],
        )->assertNotEmpty()
          ->assertMinLength(50);
    }
};

public function setUp(): void
{
    $this->variant(Lab::OpenAI, 'gpt-4o-mini');
    $this->variant(Lab::Anthropic, 'claude-sonnet-4-20250514')
        ->label('Sonnet');
}

public function setUp(): void
{
    $this->variant(Lab::OpenAI, 'gpt-4o-mini')
        ->label('Default instructions');

    $this->variant(Lab::OpenAI, 'gpt-4o-mini')
        ->label('Strict coach')
        ->instruction('You are a strict sales coach. Never offer discounts.');

    $this->variant(Lab::OpenAI, 'gpt-4o-mini')
        ->label('Lenient coach')
        ->instruction('You are a lenient sales coach. Offer discounts freely.');
}

$this->variant(Lab::OpenAI, 'gpt-4o-mini')
    ->label('Strict coach')
    ->instruction('file://prompts/strict-coach.md');

public function setUp(): void
{
    $this->variant(Lab::OpenAI, 'gpt-4o-mini')
        ->label('Precise')
        ->temperature(0.0);

    $this->variant(Lab::OpenAI, 'gpt-4o-mini')
        ->label('Creative')
        ->temperature(1.0)
        ->maxTokens(512);
}

$this->variant(Lab::OpenAI, 'gpt-4o-mini')
    ->pricing(inputPerMillion: 0.15, outputPerMillion: 0.60);

$this->variant(Lab::Anthropic, 'claude-sonnet-4-20250514')
    ->pricing(inputPerMillion: 3.00, outputPerMillion: 15.00);

'cost_resolvers' => [
    \Casawatt\LaravelAiAgentEvaluation\CostResolvers\OpenRouterCostResolver::class,
    \Casawatt\LaravelAiAgentEvaluation\CostResolvers\ModelsDevCostResolver::class,
],

use Casawatt\LaravelAiAgentEvaluation\CostResolverInterface;
use Casawatt\LaravelAiAgentEvaluation\Price;
use Laravel\Ai\Enums\Lab;

class MyCostResolver implements CostResolverInterface
{
    public function resolve(Lab|string $provider, string $model): ?Price
    {
        // Return null if this resolver doesn't handle this provider
        // Return a Price with per-million-token costs otherwise
        return new Price(inputPerMillion: 0.15, outputPerMillion: 0.60);
    }
}

$this->agent(prompt: 'Describe the product')
    ->assertStructure(['name', 'price', 'tags' => ['*' => ['label']]])
    ->assertEquals('name', 'Widget')
    ->assertContains('name', 'Wid')
    ->assertRegex('sku', '/^SKU-\d+$/')
    ->assertHasKey('price')
    ->assertMissingKey('deleted_at')
    ->assertCount(3)
    ->assertWhere('price', fn ($v) => $v > 0 && $v < 1000);

->assert(fn ($response) => str_word_count($response->text) > 10, 'Expected more than 10 words')

#[EvaluationCase]
public function evaluates_response_quality(): void
{
    $this->agent(prompt: 'What is the capital of France?')
        ->assertContains('Paris')                       // weight: 1.0 (default)
        ->assertNotContains('London')                   // weight: 1.0 (default)
        ->assertMaxLength(200, weight: 0.3);            // less important
}

#[EvaluationCase]
public function knows_capital(): void
{
    $this->agent(prompt: 'What is the capital of France?')
        ->assertContains('Paris', metric: 'accuracy')
        ->assertMinLength(20, metric: 'completeness')
        ->assertMaxLength(200, metric: 'completeness');
}

#[EvaluationCase]
public function explains_concept(): void
{
    $this->agent(prompt: 'Explain gravity in simple terms')
        ->assertContains('force', metric: 'accuracy')
        ->assertMinLength(50, metric: 'completeness')
        ->assertNotContains('kill', metric: 'safety');
}

use Casawatt\LaravelAiAgentEvaluation\Attributes\With;

#[EvaluationCase]
#[With('capitalCities')]
public function knows_capital(string $country, string $capital): void
{
    $this->agent(prompt: "What is the capital of {$country}?")
        ->assertContains($capital);
}

public function capitalCities(): array
{
    return [
        'france'  => ['France', 'Paris'],
        'germany' => ['Germany', 'Berlin'],
        'japan'   => ['Japan', 'Tokyo'],
    ];
}

public function customerQuestions(): array
{
    return Customer::query()
        ->where('type', 'test')
        ->get()
        ->mapWithKeys(fn ($c) => [$c->name => [$c->question, $c->expected_answer]])
        ->all();
}

use Casawatt\LaravelAiAgentEvaluation\Variant;

#[EvaluationCase]
public function uses_tools(): void
{
    $this->skipWhen(
        fn (Variant $v) => $v->provider === Lab::Mistral,
        'Mistral does not support tools',
    );

    $this->agent(prompt: 'Search for restaurants nearby')
        ->assertToolCalled('search');
}

#[EvaluationCase]
public function not_ready_yet(): void
{
    $this->skip('Work in progress');
}

'parallel' => 4,

// config/ai-agent-evaluation.php
'storage' => [
    'driver' => 'sqlite', // sqlite, file
    'path' => storage_path('ai-agent-evaluation'),
],

bash
php artisan vendor:publish --tag="ai-agent-evaluation-config"

bash
php artisan make:agent-evaluation SalesCoach

agent-evaluations/
    SalesCoachEvaluation.php
    results/
        .gitignore

bash
php artisan agent-evaluation

bash
# Run 4 cases in parallel
php artisan agent-evaluation --parallel=4