PHP code example of codewithkyrian / tokenizers

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

codewithkyrian / tokenizers example snippets

use Codewithkyrian\Tokenizers\Tokenizer;

// Load a tokenizer from Hugging Face Hub
$tokenizer = Tokenizer::fromHub('bert-base-uncased');

// Encode text to token IDs
$encoding = $tokenizer->encode('Hello, how are you?');

echo implode(', ', $encoding->ids);     // 101, 7592, 1010, 2129, 2024, 2017, 1029, 102
echo implode(', ', $encoding->tokens);  // [CLS], hello, ,, how, are, you, ?, [SEP]

// Decode token IDs back to text
$text = $tokenizer->decode($encoding->ids);
echo $text; // "[CLS] hello, how are you? [SEP]"

use Codewithkyrian\Tokenizers\Tokenizer;

// Load a popular model
$tokenizer = Tokenizer::fromHub('bert-base-uncased');

// Load a model from an organization
$tokenizer = Tokenizer::fromHub('meta-llama/Llama-3.1-8B-Instruct');

// With options
$tokenizer = Tokenizer::fromHub(
    modelId: 'openai/gpt-oss-20b',
    cacheDir: '/path/to/cache',      // Custom cache directory
    revision: 'main',                 // Branch, tag, or commit hash
    token: 'hf_...'                   // Auth token for private models
);

use Codewithkyrian\Tokenizers\Tokenizer;

// Single file (tokenizer.json with all config merged)
$tokenizer = Tokenizer::fromFile('/path/to/tokenizer.json');

// Multiple files (configs are merged, later files override earlier ones)
$tokenizer = Tokenizer::fromFile(
    '/path/to/tokenizer.json',
    '/path/to/tokenizer_config.json'
);

use Codewithkyrian\Tokenizers\Tokenizer;

$config = json_decode(file_get_contents('tokenizer.json'), true);
$tokenizer = Tokenizer::fromConfig($config);

use Codewithkyrian\Tokenizers\Tokenizer;

// Automatically detects the source type
$tokenizer = Tokenizer::load('bert-base-uncased');           // From Hub
$tokenizer = Tokenizer::load('/path/to/tokenizer.json');     // From file
$tokenizer = Tokenizer::load($configArray);                  // From array

$tokenizer = Tokenizer::fromHub('bert-base-uncased');

// Get a specific config value
$maxLength = $tokenizer->getConfig('model_max_length');           // 512
$cleanup = $tokenizer->getConfig('clean_up_tokenization_spaces'); // true
$custom = $tokenizer->getConfig('unknown_key', 'default');        // 'default'

// Get all configuration (pass null or no arguments)
$allConfig = $tokenizer->getConfig();

$encoding = $tokenizer->encode('The quick brown fox jumps over the lazy dog.');

$encoding->ids;      // int[] - Token IDs: [101, 1996, 4248, 2829, 4419, ...]
$encoding->tokens;   // string[] - Tokens: ['[CLS]', 'the', 'quick', 'brown', ...]
$encoding->typeIds;  // int[] - Segment IDs for sentence pairs: [0, 0, 0, ...]

$encoding = $tokenizer->encode(
    text: 'First sentence.',
    textPair: 'Second sentence.',   // Optional second text for pair encoding
    addSpecialTokens: true          // Whether to add [CLS], [SEP], etc. (default: true)
);

$encoding = $tokenizer->encode(
    text: 'What is the capital of France?',
    textPair: 'Paris is the capital of France.'
);

// tokens: ['[CLS]', 'what', 'is', 'the', 'capital', 'of', 'france', '?', '[SEP]', 
//          'paris', 'is', 'the', 'capital', 'of', 'france', '.', '[SEP]']
// typeIds: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]

$text = $tokenizer->decode([101, 7592, 1010, 2129, 2024, 2017, 1029, 102]);
// "hello, how are you?"

$text = $tokenizer->decode(
    ids: $encoding->ids,
    skipSpecialTokens: true,    // Remove [CLS], [SEP], etc. (default: true)
    cleanup: null               // Override cleanup behavior (default: use model config)
);

// With cleanup (default when model config says so)
$tokenizer->decode($ids, cleanup: true);   // "hello, how are you?"

// Without cleanup
$tokenizer->decode($ids, cleanup: false);  // "hello , how are you ?"

use Codewithkyrian\Tokenizers\Tokenizer;
use Codewithkyrian\Tokenizers\Models\WordPieceModel;
use Codewithkyrian\Tokenizers\Normalizers\LowercaseNormalizer;
use Codewithkyrian\Tokenizers\PreTokenizers\WhitespacePreTokenizer;
use Codewithkyrian\Tokenizers\PostProcessors\BertPostProcessor;
use Codewithkyrian\Tokenizers\Decoders\WordPieceDecoder;

$vocab = ['[UNK]' => 0, '[CLS]' => 1, '[SEP]' => 2, 'hello' => 3, 'world' => 4, ...];

$tokenizer = Tokenizer::builder()
    ->withModel(new WordPieceModel($vocab, '[UNK]'))
    ->withNormalizer(new LowercaseNormalizer())
    ->withPreTokenizer(new WhitespacePreTokenizer())
    ->withPostProcessor(new BertPostProcessor('[CLS]', '[SEP]'))
    ->withDecoder(new WordPieceDecoder())
    ->withSpecialTokens(['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]'])
    ->withConfig('model_max_length', 512)
    ->withConfig('clean_up_tokenization_spaces', true)
    ->build();

use Codewithkyrian\Tokenizers\Contracts\NormalizerInterface;

class CustomNormalizer implements NormalizerInterface
{
    public function normalize(string $text): string
    {
        // Your custom normalization logic
        return $modifiedText;
    }
}