PHP code example of ducks-project / encoding-repair

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

ducks-project / encoding-repair example snippets

// Custom property mapper for selective processing (60% faster!)
use Ducks\Component\EncodingRepair\Interpreter\PropertyMapperInterface;

class UserMapper implements PropertyMapperInterface
{
    public function map(object $object, callable $transcoder, array $options): object
    {
        $copy = clone $object;
        $copy->name = $transcoder($object->name);
        $copy->email = $transcoder($object->email);
        // password NOT transcoded (security)
        return $copy;
    }
}

$processor = new CharsetProcessor();
$processor->registerPropertyMapper(User::class, new UserMapper());

// Batch conversion with single encoding detection (40-60% faster!)
$rows = $db->query("SELECT * FROM users")->fetchAll();
$utf8Rows = CharsetHelper::toCharsetBatch($rows, 'UTF-8', CharsetHelper::AUTO);

// Detect encoding from array
$encoding = CharsetHelper::detectBatch($items);

// New way: Service instance
$processor = new CharsetProcessor();
$processor->addEncodings('SHIFT_JIS')->resetDetectors();
$utf8 = $processor->toUtf8($data);

// Old way: Static facade (still works)
$utf8 = CharsetHelper::toUtf8($data);

// Use built-in InternalArrayCache (default, optimized)
use Ducks\Component\EncodingRepair\Detector\CachedDetector;
use Ducks\Component\EncodingRepair\Detector\MbStringDetector;

$detector = new CachedDetector(new MbStringDetector());
// InternalArrayCache used automatically (no TTL overhead)

// Or use ArrayCache for TTL support
use Ducks\Component\EncodingRepair\Cache\ArrayCache;

$cache = new ArrayCache();
$detector = new CachedDetector(new MbStringDetector(), $cache, 3600);

// Or use any PSR-16 implementation (Redis, Memcached, APCu)
// $redis = new \Symfony\Component\Cache\Psr16Cache($redisAdapter);
// $detector = new CachedDetector(new MbStringDetector(), $redis, 7200);

use Ducks\Component\Component\EncodingRepair\CharsetHelper;

// Simple UTF-8 conversion
$utf8String = CharsetHelper::toUtf8($latinString);

// Automatic encoding detection
$data = CharsetHelper::toCharset($mixedData, 'UTF-8', CharsetHelper::AUTO);

// Repair double-encoded strings
$fixed = CharsetHelper::repair($corruptedString);

// Safe JSON with encoding handling
$json = CharsetHelper::safeJsonEncode($data);

use Ducks\Component\Component\EncodingRepair\CharsetHelper;

$data = [
    'name' => 'Gérard', // ISO-8859-1 string
    'meta' => ['desc' => 'Ca coûte 10€'] // Nested array with Euro sign
];

// Convert to UTF-8
$utf8 = CharsetHelper::toUtf8($data, CharsetHelper::WINDOWS_1252);

// Convert to ISO-8859-1 (Windows-1252)
$iso = CharsetHelper::toIso($data, CharsetHelper::ENCODING_UTF8);

// Convert to any encoding
$result = CharsetHelper::toCharset(
    $data,
    CharsetHelper::ENCODING_UTF16,
    CharsetHelper::ENCODING_UTF8
);

// Automatic detection
$result = CharsetHelper::toCharset(
    $unknownData,
    CharsetHelper::ENCODING_UTF8,
    CharsetHelper::AUTO  // Will auto-detect source encoding
);

// Manual detection
$encoding = CharsetHelper::detect($string);
echo $encoding; // "UTF-8", "ISO-8859-1", etc.

// Batch detection from array (faster for large datasets)
$encoding = CharsetHelper::detectBatch($items);

// With custom encoding list
$encoding = CharsetHelper::detect($string, [
    'encodings' => ['UTF-8', 'Shift_JIS', 'EUC-JP']
]);

// Database migration with batch processing
$rows = $db->query("SELECT * FROM users")->fetchAll(); // 10,000 rows

// Slow: Detects encoding for each row (10,000 detections)
$utf8Rows = array_map(
    fn($row) => CharsetHelper::toUtf8($row, CharsetHelper::AUTO),
    $rows
);

// Fast: Detects encoding once (1 detection, 40-60% faster!)
$utf8Rows = CharsetHelper::toCharsetBatch(
    $rows,
    CharsetHelper::ENCODING_UTF8,
    CharsetHelper::AUTO
);

// CSV import example
$csvData = array_map('str_getcsv', file('data.csv'));
$utf8Csv = CharsetHelper::toCharsetBatch($csvData, 'UTF-8', CharsetHelper::AUTO);

// Array conversion
$data = [
    'name' => 'Café',
    'city' => 'São Paulo',
    'items' => [
        'entrée' => 'Crème brûlée',
        'plat' => 'Bœuf bourguignon'
    ]
];

$utf8Data = CharsetHelper::toUtf8($data, CharsetHelper::WINDOWS_1252);

// Object conversion
class User {
    public $name;
    public $email;
}

$user = new User();
$user->name = 'José';
$user->email = 'josé@example.com';

$utf8User = CharsetHelper::toUtf8($user, CharsetHelper::ENCODING_ISO);
// Returns a cloned object with converted properties

// Example: "Café" (UTF-8 interpreted as ISO, then re-encoded as UTF-8)
$corrupted = "Café";

$fixed = CharsetHelper::repair($corrupted);
echo $fixed; // "Café"

// With custom max depth
$fixed = CharsetHelper::repair(
    $corrupted,
    CharsetHelper::ENCODING_UTF8,
    CharsetHelper::ENCODING_ISO,
    ['maxDepth' => 10]  // Try to peel up to 10 encoding layers
);

// Safe encoding (auto-repairs before encoding)
$json = CharsetHelper::safeJsonEncode($data);

// Safe decoding with charset conversion
$data = CharsetHelper::safeJsonDecode(
    $json,
    true,  // associative array
    512,   // depth
    0,     // flags
    CharsetHelper::ENCODING_UTF8,      // target encoding
    CharsetHelper::WINDOWS_1252        // source encoding for repair
);

// Throws RuntimeException on error with clear message
try {
    $json = CharsetHelper::safeJsonEncode($invalidData);
} catch (RuntimeException $e) {
    echo $e->getMessage();
    // "JSON Encode Error: Malformed UTF-8 characters"
}

$result = CharsetHelper::toCharset($data, 'UTF-8', 'ISO-8859-1', [
    'normalize' => true,   // Apply Unicode NFC normalization (default: true)
    'translit' => true,    // Transliterate unavailable chars (default: true)
    'ignore' => true,      // Ignore invalid sequences (default: true)
    'encodings' => ['UTF-8', 'ISO-8859-1', 'Shift_JIS']  // For detection
]);

use Ducks\Component\EncodingRepair\CharsetProcessor;

// Create a processor instance
$processor = new CharsetProcessor();

// Fluent API for configuration
$processor
    ->addEncodings('SHIFT_JIS', 'EUC-JP')
    ->queueTranscoders(new MyCustomTranscoder())
    ->resetDetectors();

// Use the processor
$utf8 = $processor->toUtf8($data);

// Production processor with strict encodings
$prodProcessor = new CharsetProcessor();
$prodProcessor->resetEncodings()->addEncodings('UTF-8', 'ISO-8859-1');

// Import processor with permissive encodings
$importProcessor = new CharsetProcessor();
$importProcessor->addEncodings('SHIFT_JIS', 'EUC-JP', 'GB2312');

// Both are independent
$prodResult = $prodProcessor->toUtf8($data);
$importResult = $importProcessor->toUtf8($legacyData);

use Ducks\Component\EncodingRepair\CharsetProcessorInterface;

class MyService
{
    private CharsetProcessorInterface $processor;

    public function __construct(CharsetProcessorInterface $processor)
    {
        $this->processor = $processor;
    }

    public function process($data)
    {
        return $this->processor->toUtf8($data);
    }
}

// Easy to mock in tests
$mock = $this->createMock(CharsetProcessorInterface::class);
$service = new MyService($mock);

use Ducks\Component\EncodingRepair\Interpreter\PropertyMapperInterface;

class UserMapper implements PropertyMapperInterface
{
    public function map(object $object, callable $transcoder, array $options): object
    {
        $copy = clone $object;
        $copy->name = $transcoder($object->name);
        $copy->email = $transcoder($object->email);
        // password is NOT transcoded (security)
        // avatar_binary is NOT transcoded (performance)
        return $copy;
    }
}

$processor = new CharsetProcessor();
$processor->registerPropertyMapper(User::class, new UserMapper());

$user = new User();
$user->name = 'José';
$user->password = 'secret123';  // Will NOT be converted
$utf8User = $processor->toUtf8($user);

// Performance: 60% faster for objects with 50+ properties

use Ducks\Component\EncodingRepair\Interpreter\TypeInterpreterInterface;

class ResourceInterpreter implements TypeInterpreterInterface
{
    public function supports($data): bool
    {
        return \is_resource($data);
    }

    public function interpret($data, callable $transcoder, array $options)
    {
        $content = \stream_get_contents($data);
        $converted = $transcoder($content);

        $newResource = \fopen('php://memory', 'r+');
        \fwrite($newResource, $converted);
        \rewind($newResource);

        return $newResource;
    }

    public function getPriority(): int
    {
        return 80;
    }
}

$processor->registerInterpreter(new ResourceInterpreter(), 80);

$resource = fopen('data.txt', 'r');
$convertedResource = $processor->toUtf8($resource);

use Ducks\Component\EncodingRepair\Transcoder\TranscoderInterface;

class MyCustomTranscoder implements TranscoderInterface
{
    public function transcode(string $data, string $to, string $from, array $options): ?string
    {
        if ($from === 'MY-CUSTOM-ENCODING') {
            return myCustomConversion($data, $to);
        }
        // Return null to try next transcoder in chain
        return null;
    }

    public function getPriority(): int
    {
        return 75; // Between iconv (50) and UConverter (100)
    }

    public function isAvailable(): bool
    {
        return extension_loaded('my_extension');
    }
}

// Register with default priority
CharsetHelper::registerTranscoder(new MyCustomTranscoder());

// Register with custom priority
CharsetHelper::registerTranscoder(new MyCustomTranscoder(), 150);

// Legacy: Register a callable
CharsetHelper::registerTranscoder(
    function (string $data, string $to, string $from, array $options): ?string {
        if ($from === 'MY-CUSTOM-ENCODING') {
            return myCustomConversion($data, $to);
        }
        return null;
    },
    150  // Priority
);

use Ducks\Component\EncodingRepair\Detector\DetectorInterface;

class MyCustomDetector implements DetectorInterface
{
    public function detect(string $string, array $options): ?string
    {
        // Check for UTF-16LE BOM
        if (strlen($string) >= 2 && ord($string[0]) === 0xFF && ord($string[1]) === 0xFE) {
            return 'UTF-16LE';
        }
        // Return null to try next detector
        return null;
    }

    public function getPriority(): int
    {
        return 150; // Higher than MbStringDetector (100)
    }

    public function isAvailable(): bool
    {
        return true;
    }
}

// Register with default priority
CharsetHelper::registerDetector(new MyCustomDetector());

// Register with custom priority
CharsetHelper::registerDetector(new MyCustomDetector(), 200);

// Legacy: Register a callable
CharsetHelper::registerDetector(
    function (string $string, array $options): ?string {
        if (strlen($string) >= 2 && ord($string[0]) === 0xFF && ord($string[1]) === 0xFE) {
            return 'UTF-16LE';
        }
        return null;
    },
    200  // Priority
);

// Default: InternalArrayCache (optimized, no TTL overhead)
$detector = new CachedDetector(new MbStringDetector());

// With TTL: ArrayCache
$cache = new ArrayCache();
$detector = new CachedDetector(new MbStringDetector(), $cache, 3600);

// External: Redis, Memcached, APCu, etc.
// $redis = new \Symfony\Component\Cache\Psr16Cache($redisAdapter);
// $detector = new CachedDetector(new MbStringDetector(), $redis, 7200);

// Migrate user table
$users = $db->query("SELECT * FROM users")->fetchAll();

foreach ($users as $user) {
    $user = CharsetHelper::toUtf8($user, CharsetHelper::ENCODING_ISO);
    $db->update('users', $user, ['id' => $user['id']]);
}

$csv = file_get_contents('data.csv');

// Auto-detect and convert
$utf8Csv = CharsetHelper::toCharset(
    $csv,
    CharsetHelper::ENCODING_UTF8,
    CharsetHelper::AUTO
);

// Parse as UTF-8
$data = str_getcsv($utf8Csv);

// Ensure API responses are always valid UTF-8
class ApiController
{
    public function jsonResponse($data): JsonResponse
    {
        $json = CharsetHelper::safeJsonEncode($data);
        return new JsonResponse($json, 200, [], true);
    }
}

$html = file_get_contents('https://example.com');

// Detect encoding from HTML meta tags or auto-detect
$encoding = CharsetHelper::detect($html);

// Convert to UTF-8 for processing
$utf8Html = CharsetHelper::toCharset(
    $html,
    CharsetHelper::ENCODING_UTF8,
    $encoding
);

$dom = new DOMDocument();
$dom->loadHTML($utf8Html);

// Fix double-encoded data from old system
$legacyData = $oldSystem->getData();

// Repair corruption
$clean = CharsetHelper::repair(
    $legacyData,
    CharsetHelper::ENCODING_UTF8,
    CharsetHelper::ENCODING_ISO
);

// Process clean data
processData($clean);

bash
# Ubuntu/Debian
sudo apt-get install php-intl php-iconv

# macOS (via Homebrew)
brew install [email protected]
# Extensions are