PHP code example of devtronic / super-tokenizer

1. Go to this page and download the library: Download devtronic/super-tokenizer library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

devtronic / super-tokenizer example snippets




use Devtronic\SuperTokenizer\Tokenizer;

mal tokenizer example';

$tokens = $tokenizer->tokenize($sample);
print_r($tokens);



// ...
foreach ($tokens as &$token) {
    $token['name'] = $tokenizer->getTokenName($token['type']);
}

print_r($tokens);




use Devtronic\SuperTokenizer\SimpleTokenizer;

ple" \'Tokenizer\' with\ different brackets [a, b] (c,d), {0, 1}';

$tokens = $tokenizer->tokenize($sample);

foreach ($tokens as &$token) {
    $token['name'] = $tokenizer->getTokenName($token['type']);
}

print_r($tokens);




use Devtronic\SuperTokenizer\SimpleTokenizer;

const TT_DOLLAR = 30;
    const TT_EQUALS = 35;

    public function __construct()
    {
        parent::__construct();

        $this->customTokens = [
            self::TT_DOLLAR => '$',
            self::TT_EQUALS => '='
        ];
    }
}

$tokenizer = new CustomTokenizer();

$sample = '$var = 1234';
$tokens = $tokenizer->tokenize($sample);

foreach ($tokens as &$token) {
    $token['name'] = $tokenizer->getTokenName($token['type']);
}

print_r($tokens);