PHP code example of affinity4 / tokenizer

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

affinity4 / tokenizer example snippets

$template = <<<TEMPLATE
html(lang="en_IE") {
    // child nodes are inside curly brackets!
    head() {
        title(): This is a title;
        link(src="./style.css");
        script(src="./main.js");
    }

    body.app#app() {
        h1.title(): Page title;
    }
}
TEMPLATE;

$lexicon = [
    /*
    It's a good idea to do the punctuation first, or anything you want to remove early on (e.g. comments or whitespace)
    This would be single chars that have meaning in your language. 
    For us, the # means an id attribute, the . is before any classname, 
    and :, ;, (, ), {, } all have their own purpose too
    */
    'T_WHITESPACE' => '\s+', // We might want to remove all whitespace not within quotes ("") to minify our compiled html
    'T_FORWARD_SLASH' => '/',
    'T_DOT' => '\.',
    'T_HASH' => '#',
    'T_COLON' => ':',
    'T_SEMICOLON' => ';',
    'T_EQUALS' => '=',
    'T_DOUBLE_QOUTE' => '"',
    'T_SINGLE_QUOTE' => "'",
    'T_EXCLAIMATION_MARK' => '!',
    'T_OPEN_PARENTHESIS' => '\(',
    'T_CLOSE_PARENTHESIS' => '\)',
    'T_OPEN_CURLY' => '\{',
    'T_CLOSE_CURLY' => '\}',

    // Now we can define some more generic "lexemes"
    
    // Match All words as T_STRING. Our parser can then 
    // check for the first string in each line that is followed by 
    // T_DOT | T_HASH | T_OPENING_PARENTHESIS. This will be the HTML tag name
    'T_STRING' => "\w+"
];

use Affinity4\Tokenizer\Tokenizer;

$Tokenizer = new Tokenizer($template);
$Tokenizer->registerLexicon($lexicon);
$Stream = $Tokenizer->tokenize(); // Affinity4\Tokenizer\Stream of Affinity4\Tokenizer\Token objects

while ($Stream->hasNext()) {
    $Token = $Stream->nextToken();
    echo $Token->type; // T_HTML_TAG
    echo $Token->value; // html
    echo $Token->linenumber; // 1
    echo $Token->offset[0]; // 0 Start position of match
    echo $Token->offset[1]; // 3 End position of match
    
}

$lexicon = [
    // ...
    Tokenize::T_ESCAPE_CHAR => 'T_ESCAPE', // '\\\\'
    Tokenize::T_NEWLINE     => 'T_NEWLINE', // ';T_NEWLINE;'
    // ...
]

$lexicon = [
    // ...
    Tokenize::T_NEWLINE => 'T_NEWLINE', // ';T_NEWLINE;'
    // ...
]

$input = str_replace(["\r\n", "\r", "\n"], ";T_NEWLINE;", $input);
$input = str_replace("\t", ";T_TAB;", $input);

$lexicon = [
    // ...
    Token::T_NEWLINE => 'T_NEWLINE',
    Token::T_TAB     => 'T_TAB'
    // ...
];

$Tokenizer = new Tokenizer($lexicon);
$Steam = $Tokenizer->tokenize($input);