PHP code example of nuglif / chunkytown

1. Go to this page and download the library: Download nuglif/chunkytown library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

nuglif / chunkytown example snippets


use ChunkyTown\Chunker;
use ChunkyTown\ChunkerConfig;

$text = file_get_contents('article.txt');

$chunker = new Chunker();
$cfg = new ChunkerConfig(
    chunkSize: 1500,   // target graphemes per chunk
    window: 175,       // ± snapping window around the chunk end
    overlap: 275,      // desired overlap between consecutive chunks
    overlapWindow: 150 // ± snapping window when looking for the overlap start
);

$spans = $chunker->chunk($text, $cfg);
// $spans is a list of [start, end) tuples in grapheme offsets.

foreach ($chunker->render($text, $spans) as $i => $chunk) {
    echo "=== CHUNK " . ($i + 1) . " ===\n$chunk\n\n";
}


$analysis = $chunker->analyze($text);

$spansA = $chunker->chunk($analysis, new ChunkerConfig(chunkSize: 1000));
$spansB = $chunker->chunk($analysis, new ChunkerConfig(chunkSize: 2000));
// boundary detection runs once, snapping runs twice.


$spans = $chunker->chunk($text, $cfg, externalBoundaries: [
    'sentence' => [120, 305, 480],
    'paragraph' => [0, 480, 1100],
]);