PHP code example of rahul900day / tiktoken-php

1. Go to this page and download the library: Download rahul900day/tiktoken-php library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

rahul900day / tiktoken-php example snippets


use Rahul900day\Tiktoken\Tiktoken;

$encoder = Tiktoken::getEncodingForModel('gpt-4');
$encoder->encode("hello world aaaaaaaaaaaa");
$encoder->decode([9906, 4435]);


use Rahul900day\Tiktoken\Tiktoken;

$encoder = Tiktoken::getEncodingForModel('gpt-4');
$encoder->encode('<|endoftext|>', allowedSpecial: 'all');


use Rahul900day\Tiktoken\Encodings\OpenAiPublic\Cl100KBaseEncoding;

class Cl100KIm extends Cl100KBaseEncoding 
{
    protected function getName(): string
    {
        return 'cl100k_im';
    }
    
    protected function getSpecialTokens(): array
    {
        return [
            ...parent::getSpecialTokens(),
            "<|im_start|>" => 100264,
            "<|im_end|>" => 100265,
        ];
    }
}

use Rahul900day\Tiktoken\Registry;
use Rahul900day\Tiktoken\Tiktoken;

Registry::registerCustomEncoding('cl100k_im', new Cl100KIm);
$encoding = Tiktoken::getEncoding('cl100k_im');

// Expect: 100264
$encoding->encode("<|im_start|>", allowedSpecial: 'all');


shell
composer