1. Go to this page and download the library: Download rahul900day/tiktoken-php library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
rahul900day / tiktoken-php example snippets
use Rahul900day\Tiktoken\Tiktoken;
$encoder = Tiktoken::getEncodingForModel('gpt-4');
$encoder->encode("hello world aaaaaaaaaaaa");
$encoder->decode([9906, 4435]);
use Rahul900day\Tiktoken\Tiktoken;
$encoder = Tiktoken::getEncodingForModel('gpt-4');
$encoder->encode('<|endoftext|>', allowedSpecial: 'all');
use Rahul900day\Tiktoken\Encodings\OpenAiPublic\Cl100KBaseEncoding;
class Cl100KIm extends Cl100KBaseEncoding
{
protected function getName(): string
{
return 'cl100k_im';
}
protected function getSpecialTokens(): array
{
return [
...parent::getSpecialTokens(),
"<|im_start|>" => 100264,
"<|im_end|>" => 100265,
];
}
}
use Rahul900day\Tiktoken\Registry;
use Rahul900day\Tiktoken\Tiktoken;
Registry::registerCustomEncoding('cl100k_im', new Cl100KIm);
$encoding = Tiktoken::getEncoding('cl100k_im');
// Expect: 100264
$encoding->encode("<|im_start|>", allowedSpecial: 'all');
shell
composer
Loading please wait ...
Before you can download the PHP files, the dependencies should be resolved. This can take some minutes. Please be patient.