1. Go to this page and download the library: Download tanuel/tokenizer library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
tanuel / tokenizer example snippets
// Token Definitions must start with T_, otherwise they won't be interpreted as Tokens
use \Tanuel\Tokenizer\AbstractTokenDefinition;
// The BaseTokenInterface is optional, but will provide some basic utilities
use \Tanuel\Tokenizer\BaseTokenInterface;
class T extends AbstractTokenDefinition implements BaseTokenInterface
{
/**
* A single dollar sign
* @pattern \$
*/
const T_DOLLAR = 'T_DOLLAR';
/**
* A range of digits
* @pattern \d+
*/
const T_DIGITS = 'T_DIGITS';
}
// using the token definition from above
$tokenizer = new \Tanuel\Tokenizer\Tokenizer('string to tokenize $ 123', T::class);
// get all tokens
$t = $tokenizer->getAll();
// reset internal pointer
$tokenizer->reset();
// get next token, ignoring leading whitespaces and linebreaks (T_WHITESPACE => \s+)
$token = $tokenizer->next();
// get next token, don't ignore leading whitespaces or linebreaks
$token = $tokenizer->next(false);
// expect a certain set of tokens, else throw an exception
try {
$token = $tokenizer->nextOf([T::T_DOLLAR, T::T_DIGITS]);
} catch (\Tanuel\Tokenizer\TokenizerException $e) {
// do something with exception
}
// forecast next token without moving the pointer forward
$forecast = $tokenizer->forecast();
// also works with expecting a certain token
$forecast = $tokenizer->forecastOf([T::T_DOLLAR, T::T_DIGITS]);
/** @var $token \Tanuel\Tokenizer\Token */
// get the matched value
$token->getValue();
// check if the token matches a certain name
$token->eq('T_STRING', 'T_DOLLAR'); // true if it is a T_STRING or T_DOLLAR
// get the token definition info
$token->getDefinition()->getName(); // e.g. T_STRING
$token->getDefinition()->getPattern(); // e.g. \w+ for T_STRING
// get metainfo
$token->getLine();
$token->getEndLine();
$token->getColumn();
$token->getEndColumn();
$token->getLineCount();
Loading please wait ...
Before you can download the PHP files, the dependencies should be resolved. This can take some minutes. Please be patient.