PHP code example of token27 / nexus-ai-tokenizer

1. Go to this page and download the library: Download token27/nexus-ai-tokenizer library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

token27 / nexus-ai-tokenizer example snippets


use Token27\Tokenizer\Engine\TokenizerEngine;

$text = 'Hello world! This is a test of the token counting library.';

// Automatically resolves 'gpt-4o' to Tiktoken (o200k_base)
$count = TokenizerEngine::for('gpt-4o')->count($text);

echo $count->count(); // 14
echo $count->strategy(); // tiktoken:o200k_base
echo $count->isApproximate() ? 'Yes' : 'No'; // No


$messages = [
    ['role' => 'system', 'content' => 'You are a helpful assistant.'],
    ['role' => 'user', 'content' => 'What is tokenization?'],
];

$chat = TokenizerEngine::for('gpt-4-turbo')->countChat($messages);

echo $chat->contentTokens();  // 12
echo $chat->overheadTokens(); // 11
echo $chat->count();          // 23


$count = TokenizerEngine::for('claude-3-opus-20240229')->count($hugeDocument);
$limit = 200_000;

if ($count->isWithinContextWindow($limit)) {
    echo "Fits! " . $count->remainingTokens($limit) . " tokens left.";
} else {
    echo "Too large by " . ($count->count() - $limit) . " tokens.";
}


// OpenAI High-detail image token math
$imageCost = TokenizerEngine::for('gpt-4o')->estimateImage(1920, 1080, 'high');
echo $imageCost->count(); // e.g. 1105 tokens

// Anthropic logic
$claudeCost = TokenizerEngine::for('claude-sonnet-4-20250514')->estimateImage(1920, 1080);
echo $claudeCost->count(); // e.g. 2765 tokens