PHP code example of danny50610 / bpe-tokeniser

1. Go to this page and download the library: Download danny50610/bpe-tokeniser library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

danny50610 / bpe-tokeniser example snippets


use Danny50610\BpeTokeniser\EncodingFactory;

$enc = EncodingFactory::createByEncodingName('cl100k_base');

var_dump($enc->encode("hello world"));
/**
 * output: 
 * array(2) {
 *  [0]=>
 *  int(15339)
 *  [1]=>
 *  int(1917)
 * }
 */

var_dump($enc->decode($enc->encode("hello world")));
// output: string(11) "hello world"


use Danny50610\BpeTokeniser\EncodingFactory;

$enc = EncodingFactory::createByModelName('gpt-3.5-turbo');

var_dump($enc->decode($enc->encode("hello world")));
// output: string(11) "hello world"