PHP code example of nette / tokenizer

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

nette / tokenizer example snippets

$tokenizer = new Nette\Tokenizer\Tokenizer([
	T_DNUMBER => '\d+',
	T_WHITESPACE => '\s+',
	T_STRING => '\w+',
]);

$stream = $tokenizer->tokenize("say \n123");

[
	new Token('say', T_STRING, 0),
	new Token(" \n", T_WHITESPACE, 3),
	new Token('123', T_DNUMBER, 5),
]

$firstToken = $stream->tokens[0];
$firstToken->value; // say
$firstToken->type; // value of T_STRING
$firstToken->offset; // position in string: 0

$input = '
	@author David Grudl
	@package Nette
';

use Nette\Tokenizer\Tokenizer;
use Nette\Tokenizer\Stream;

class Parser
{
	const T_AT = 1;
	const T_WHITESPACE = 2;
	const T_STRING = 3;

	/** @var Tokenizer */
	private $tokenizer;

	/** @var Stream */
	private $stream;

	public function __construct()
	{
		$this->tokenizer = new Tokenizer([
			self::T_AT => '@',
			self::T_WHITESPACE => '\s+',
			self::T_STRING => '\w+',
		]);
	}

	public function parse(string $input): array
	{
		$this->stream = $this->tokenizer->tokenize($input);

		$result = [];
		while ($this->stream->nextToken()) {
			if ($this->stream->isCurrent(self::T_AT)) {
				$result[] = $this->parseAnnotation();
			}
		}

		return $result;
	}

	private function parseAnnotation(): array
	{
		$name = $this->stream->joinUntil(self::T_WHITESPACE);
		$this->stream->nextUntil(self::T_STRING);
		$content = $this->stream->joinUntil(self::T_AT);

		return [$name, trim($content)];
	}
}

$parser = new Parser;
$annotations = $parser->parse($input);

// iterate until a string or a whitespace is found, then return the following token
$token = $stream->nextToken(T_STRING, T_WHITESPACE);

// give me next token
$token = $stream->nextToken();

// move the cursor until you find token containing only '@', then stop and return it
$token = $stream->nextToken('@');

// is the current token '@' or type of T_AT?
$stream->isCurrent(T_AT, '@');

array (2)
   0 => array (2)
   |  0 => 'author'
   |  1 => 'David Grudl'
   1 => array (2)
   |  0 => 'package'
   |  1 => 'Nette'