1. Go to this page and download the library: Download everstu/gpt3-tokenizer library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
everstu / gpt3-tokenizer example snippets
use everstu\Gpt3Tokenizer\Gpt3TokenizerConfig;
// default vocab path
// default merges path
// caching enabled
$defaultConfig = new Gpt3TokenizerConfig();
$customConfig = new Gpt3TokenizerConfig();
$customConfig
->vocabPath('custom_vocab.json') // path to a custom vocabulary file
->mergesPath('custom_merges.txt') // path to a custom merges file
->useCache(false)
use everstu\Gpt3Tokenizer\Gpt3TokenizerConfig;
use everstu\Gpt3Tokenizer\Gpt3Tokenizer;
$config = new Gpt3TokenizerConfig();
$tokenizer = new Gpt3Tokenizer($config);
$text = "This is some text";
$tokens = $tokenizer->encode($text);
// [1212,318,617,2420]
use everstu\Gpt3Tokenizer\Gpt3TokenizerConfig;
use everstu\Gpt3Tokenizer\Gpt3Tokenizer;
$config = new Gpt3TokenizerConfig();
$tokenizer = new Gpt3Tokenizer($config);
$tokens = [1212,318,617,2420]
$text = $tokenizer->decode($tokens);
// "This is some text"
use everstu\Gpt3Tokenizer\Gpt3TokenizerConfig;
use everstu\Gpt3Tokenizer\Gpt3Tokenizer;
$config = new Gpt3TokenizerConfig();
$tokenizer = new Gpt3Tokenizer($config);
$text = "This is some text";
$numberOfTokens = $tokenizer->count($text);
// 4
Loading please wait ...
Before you can download the PHP files, the dependencies should be resolved. This can take some minutes. Please be patient.