PHP code example of yaroslavpopovic / laravel-text-chunker

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

yaroslavpopovic / laravel-text-chunker example snippets

return [
    // Default strategy to use when none is specified
    'default_strategy' => 'character',

    // Strategy-specific configurations
    'strategies' => [
        'token' => [
            // Default OpenAI model for token encoding
            'model' => 'gpt-4',
        ],
        'sentence' => [
            // Abbreviations that should not trigger sentence breaks
            'abbreviations' => ['Dr', 'Mr', 'Mrs', 'Ms', 'Prof', 'Sr', 'Jr'],
        ],
    ],

    // Register custom strategies here
    'custom_strategies' => [
        // 'my-strategy' => \App\TextChunking\MyCustomStrategy::class,
    ],
];

use Droath\TextChunker\Facades\TextChunker;

$text = "Your long text content here...";

$chunks = TextChunker::strategy('character')
    ->size(100)
    ->chunk($text);

foreach ($chunks as $chunk) {
    echo "Chunk {$chunk->index}: {$chunk->text}\n";
    echo "Position: {$chunk->start_position} to {$chunk->end_position}\n";
}

use Droath\TextChunker\Facades\TextChunker;

$text = "Your long text content here...";

$chunks = TextChunker::strategy('token')
    ->size(500) // 500 tokens per chunk
    ->chunk($text);

// Use different OpenAI model for encoding
$chunks = TextChunker::strategy('token', ['model' => 'gpt-3.5-turbo'])
    ->size(500)
    ->chunk($text);

use Droath\TextChunker\Facades\TextChunker;

$text = "First sentence. Second sentence. Third sentence.";

$chunks = TextChunker::strategy('sentence')
    ->size(2) // 2 sentences per chunk
    ->chunk($text);

// Custom abbreviations
$chunks = TextChunker::strategy('sentence', [
        'abbreviations' => ['Dr', 'Mr', 'Mrs', 'Ph.D']
    ])
    ->size(3)
    ->chunk($text);

use Droath\TextChunker\Facades\TextChunker;

$markdown = <<<'MD'
# Heading 1

Some content here.

use Droath\TextChunker\Facades\TextChunker;

$text = "Your long text content here...";

$chunks = TextChunker::strategy('character')
    ->size(100)
    ->overlap(20) // 20% overlap between chunks
    ->chunk($text);

// Each chunk will 

$chunks = TextChunker::strategy('character')->size(100)->chunk($text);

foreach ($chunks as $chunk) {
    $chunk->text;             // The chunk text content
    $chunk->index;            // Zero-based index (0, 1, 2, ...)
    $chunk->start_position;   // Character offset in original text (inclusive)
    $chunk->end_position;     // Character offset in original text (exclusive)
}

use Droath\TextChunker\TextChunkerManager;

class MyService
{
    public function __construct(
        protected TextChunkerManager $chunker
    ) {}

    public function processText(string $text): array
    {
        return $this->chunker
            ->strategy('token')
            ->size(500)
            ->overlap(10)
            ->chunk($text);
    }
}

declare(strict_types=1);

namespace App\TextChunking;

use Droath\TextChunker\DataObjects\Chunk;
use Droath\TextChunker\Concerns\HasOverlap;
use Droath\TextChunker\Contracts\ChunkerStrategyInterface;

class WordStrategy implements ChunkerStrategyInterface
{
    use HasOverlap; // Optional: for overlap support

    public function chunk(string $text, int $size, array $options): array
    {
        $words = explode(' ', $text);
        $chunks = [];
        $index = 0;
        $position = 0;

        foreach (array_chunk($words, $size) as $wordChunk) {
            $chunkText = implode(' ', $wordChunk);
            $chunkLength = mb_strlen($chunkText);

            $chunks[] = new Chunk(
                text: $chunkText,
                index: $index++,
                start_position: $position,
                end_position: $position + $chunkLength
            );

            $position += $chunkLength + 1; // +1 for space
        }

        return $chunks;
    }
}

return [
    'custom_strategies' => [
        'word' => \App\TextChunking\WordStrategy::class,
    ],
];

use Droath\TextChunker\Facades\TextChunker;
use App\TextChunking\WordStrategy;

TextChunker::extend('word', WordStrategy::class);

$chunks = TextChunker::strategy('word')->size(50)->chunk($text);

use Droath\TextChunker\TextChunkerManager;
use App\TextChunking\WordStrategy;

public function boot(TextChunkerManager $chunker): void
{
    $chunker->extend('word', WordStrategy::class);
}

TextChunker::strategy(string $name, array $options = [])  // Select strategy
    ->size(int $size)                                      // Set chunk size
    ->overlap(int $percentage)                             // Set overlap (0-100)
    ->chunk(string $text)                                  // Execute and return chunks

use Droath\TextChunker\Facades\TextChunker;
use Droath\TextChunker\Exceptions\ChunkerException;

try {
    $chunks = TextChunker::strategy('character')
        ->size(100)
        ->overlap(150) // Invalid: must be 0-100
        ->chunk($text);
} catch (ChunkerException $e) {
    // Handle validation error
    echo $e->getMessage(); // "Overlap percentage must be between 0 and 100"
}

bash
php artisan vendor:publish --tag="text-chunker-config"

bash
composer analyse