PHP code example of spiritinlife / php-mapreduce

1. Go to this page and download the library: Download spiritinlife/php-mapreduce library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

spiritinlife / php-mapreduce example snippets


use Spiritinlife\MapReduce\MapReduceBuilder;

$documents = [
    'doc1' => 'the quick brown fox',
    'doc2' => 'the lazy dog',
    'doc3' => 'quick brown dogs',
];

$results = (new MapReduceBuilder())
    ->input($documents)
    ->map(function ($text, $context) {
        foreach (str_word_count(strtolower($text), 1) as $word) {
            yield [$word, 1];
        }
    })
    ->reduce(function ($word, $countsIterator, $context) {
        $sum = 0;
        foreach ($countsIterator as $count) {
            $sum += $count;
        }
        return $sum;
    })
    ->execute();

// Results are streamed as a generator - iterate to process results
foreach ($results as $word => $count) {
    // Output: 'quick' => 2, 'brown' => 2, 'the' => 2, ...
}

// Simple array
->input(['a', 'b', 'c'])

// Associative array
->input(['doc1' => 'text', 'doc2' => 'more text'])

// ArrayIterator
->input(new ArrayIterator($data))

// Read file line by line (memory-efficient)
->input(new SplFileObject('large-file.txt'))

// Generator from file
->input((function() {
    $handle = fopen('data.csv', 'r');
    while (($line = fgets($handle)) !== false) {
        yield $line;
    }
    fclose($handle);
})())

// Parse CSV rows
->input((function() {
    $file = fopen('data.csv', 'r');
    fgetcsv($file); // Skip header
    while (($row = fgetcsv($file)) !== false) {
        yield [
            'name' => $row[0],
            'value' => $row[1],
        ];
    }
    fclose($file);
})())

// Process newline-delimited JSON
->input((function() {
    $file = fopen('data.jsonl', 'r');
    while (($line = fgets($file)) !== false) {
        yield json_decode($line, true);
    }
    fclose($file);
})())

// PDO result set
$stmt = $pdo->query('SELECT * FROM large_table');
->input($stmt)

// Generator for memory efficiency
->input((function() use ($pdo) {
    $stmt = $pdo->prepare('SELECT * FROM users');
    $stmt->execute();
    while ($row = $stmt->fetch()) {
        yield $row;
    }
})())

// Process all files in a directory
->input(new RecursiveIteratorIterator(
    new RecursiveDirectoryIterator('/path/to/logs')
))

// Custom file filter
->input((function() {
    $dir = new RecursiveDirectoryIterator('/path/to/data');
    $iterator = new RecursiveIteratorIterator($dir);
    foreach ($iterator as $file) {
        if ($file->isFile() && $file->getExtension() === 'log') {
            yield $file->getPathname() => file_get_contents($file);
        }
    }
})())

->input($documents)
->input(new ArrayIterator($data))

->map(function ($value, $context) {
    // Process and emit key-value pairs
    yield [$newKey, $newValue];
})

// Accumulator pattern - iterate and accumulate manually
->reduce(function ($key, $valuesIterator, $context) {
    $sum = 0;  // Accumulator
    foreach ($valuesIterator as $value) {
        $sum += $value;  // Accumulate
    }
    return $sum;
})

// Using context data
->reduce(function ($key, $valuesIterator, $context) {
    $threshold = $context['threshold'];
    $sum = 0;
    foreach ($valuesIterator as $value) {
        if ($value > $threshold) {
            $sum += $value;
        }
    }
    return $sum;
})

// If you need the full array (uses memory):
->reduce(function ($key, $valuesIterator, $context) {
    $values = iterator_to_array($valuesIterator);
    return ['count' => count($values), 'unique' => array_unique($values)];
})

// ❌ This doesn't work - variables won't be available in parallel processes
$threshold = 100;
->map(function ($value, $context) use ($threshold) {  // Won't work!
    if ($value > $threshold) yield [$value, 1];
})

// ✅ Use context() instead
->context(['threshold' => 100])
->map(function ($value, $context) {
    if ($value > $context['threshold']) yield [$value, 1];
})

->concurrent(8)  // Spawn 8 parallel PHP processes

->partitions(16)  // More parallelism in reduce phase

->mapperBatchSize(2000)   // Large datasets - reduce process overhead
->mapperBatchSize(100)    // Small datasets - better load balancing

->shuffleChunkSize(50000)   // High-memory system - faster sorting
->shuffleChunkSize(2000)    // Memory-constrained

->bufferSize(5000)   // Large datasets, plenty of RAM
->bufferSize(500)    // Memory-constrained

->workingDirectory('/mnt/fast-ssd/tmp')

->autoload(__DIR__ . '/vendor/autoload.php')

->partitionBy(function ($key, $numPartitions, $context) {
    return ord($key[0]) % $numPartitions;  // Partition by first letter
})

$sales = [
    ['product' => 'Widget', 'amount' => 100],
    ['product' => 'Gadget', 'amount' => 150],
    ['product' => 'Widget', 'amount' => 200],
];

$totals = (new MapReduceBuilder())
    ->input($sales)
    ->map(fn($sale, $context) => yield [$sale['product'], $sale['amount']])
    ->reduce(function($product, $amountsIterator, $context) {
        // Collect amounts to calculate statistics
        $amounts = iterator_to_array($amountsIterator);
        return [
            'total' => array_sum($amounts),
            'average' => array_sum($amounts) / count($amounts),
            'count' => count($amounts),
        ];
    })
    ->execute();

foreach ($totals as $product => $stats) {
    echo "{$product}: {$stats['total']}\n";
}

$documents = [
    ['id' => 1, 'text' => 'the quick brown fox'],
    ['id' => 2, 'text' => 'the lazy dog'],
    ['id' => 3, 'text' => 'quick brown animals'],
];

$invertedIndex = (new MapReduceBuilder())
    ->input($documents)
    ->map(function ($doc, $context) {
        foreach (array_unique(str_word_count(strtolower($doc['text']), 1)) as $word) {
            yield [$word, $doc['id']];
        }
    })
    ->reduce(function($word, $docIdsIterator, $context) {
        // Collect document IDs to calculate frequency and uniqueness
        $docIds = iterator_to_array($docIdsIterator);
        return [
            'documents' => array_unique($docIds),
            'frequency' => count($docIds),
        ];
    })
    ->execute();

foreach ($invertedIndex as $word => $index) {
    echo "{$word}: appears in {$index['frequency']} documents\n";
}

$stats = (new MapReduceBuilder())
    ->input(file('access.log'))
    ->map(function ($line, $context) {
        if (preg_match('/^(\S+).*?"GET (\S+).*?" (\d+)/', $line, $m)) {
            yield ["ip:{$m[1]}", 1];
            yield ["status:{$m[3]}", 1];
        }
    })
    ->reduce(function($key, $countsIterator, $context) {
        $sum = 0;
        foreach ($countsIterator as $count) {
            $sum += $count;
        }
        return $sum;
    })
    ->concurrent(8)
    ->execute();

foreach ($stats as $key => $count) {
    echo "{$key}: {$count}\n";
}

// Pre-calculated lookup tables or configuration
$userTiers = ['alice' => 'gold', 'bob' => 'silver', 'charlie' => 'bronze'];
$tierMultipliers = ['gold' => 1.5, 'silver' => 1.2, 'bronze' => 1.0];

$purchases = [
    ['user' => 'alice', 'amount' => 100],
    ['user' => 'bob', 'amount' => 150],
    ['user' => 'charlie', 'amount' => 200],
];

$results = (new MapReduceBuilder())
    ->input($purchases)
    // Pass shared data via context (not via 'use' clause!)
    ->context([
        'userTiers' => $userTiers,
        'tierMultipliers' => $tierMultipliers,
    ])
    ->map(function ($purchase, $context) {
        // Access context data in mapper
        $tier = $context['userTiers'][$purchase['user']] ?? 'bronze';
        $multiplier = $context['tierMultipliers'][$tier];
        $bonusPoints = $purchase['amount'] * $multiplier;

        yield [$purchase['user'], $bonusPoints];
    })
    ->reduce(function ($user, $pointsIterator, $context) {
        // Context also available in reducer
        $tier = $context['userTiers'][$user];

        // Stream and sum points without loading all into memory
        $totalPoints = 0;
        foreach ($pointsIterator as $points) {
            $totalPoints += $points;
        }

        return [
            'total_points' => $totalPoints,
            'tier' => $tier,
        ];
    })
    ->execute();

foreach ($results as $user => $data) {
    echo "{$user} ({$data['tier']}): {$data['total_points']} points\n";
}

// ❌ BROKEN - Variables won't be available in parallel processes
$totalUsers = 1000;
$lookupTable = ['a' => 1, 'b' => 2];

(new MapReduceBuilder())
    ->input($data)
    ->map(function ($item, $context) use ($lookupTable) {
        // ❌ $lookupTable will be NULL or empty here!
        $value = $lookupTable[$item['key']];  // Won't work!
        yield [$item['key'], $value];
    })
    ->reduce(function ($key, $valuesIterator, $context) use ($totalUsers) {
        // ❌ $totalUsers will be NULL or 0 here!
        $sum = 0;
        foreach ($valuesIterator as $value) {
            $sum += $value;
        }
        $percentage = $sum / $totalUsers;  // Won't work!
        return $percentage;
    })
    ->execute();

// ✅ CORRECT - Use context() to pass data
$totalUsers = 1000;
$lookupTable = ['a' => 1, 'b' => 2];

(new MapReduceBuilder())
    ->input($data)
    ->context([
        'totalUsers' => $totalUsers,
        'lookupTable' => $lookupTable,
        'config' => ['threshold' => 50]
    ])
    ->map(function ($item, $context) {
        // ✅ Access via $context parameter
        $value = $context['lookupTable'][$item['key']];
        yield [$item['key'], $value];
    })
    ->reduce(function ($key, $valuesIterator, $context) {
        // ✅ Context available in reducer too
        $sum = 0;
        foreach ($valuesIterator as $value) {
            $sum += $value;
        }
        $percentage = $sum / $context['totalUsers'];
        return $percentage;
    })
    ->execute();

// Phase 1: Calculate counts (stored in memory)
$totalBuyers = 10000;
$eventCounts = [
    'follow' => ['seller1' => 500, 'seller2' => 300],
    'purchase' => ['seller1' => 200, 'seller2' => 150],
];

// Phase 2: Use counts to calculate scores with MapReduce
$results = (new MapReduceBuilder())
    ->input($cooccurrences)
    ->context([
        'totalBuyers' => $totalBuyers,
        'eventCounts' => $eventCounts,
        'minScore' => 5.0
    ])
    ->reduce(function ($key, $valuesIterator, $context) {
        // Extract shared data from context
        $totalBuyers = $context['totalBuyers'];
        $eventCounts = $context['eventCounts'];

        // Collect values for calculation
        $values = iterator_to_array($valuesIterator);

        // Use in calculations
        $score = calculateScore($values, $totalBuyers, $eventCounts);

        if ($score >= $context['minScore']) {
            return ['score' => $score, 'count' => count($values)];
        }

        return null;
    })
    ->execute();

(new MapReduceBuilder())
    ->concurrent(8)                  // CPU cores (or 2-3x for I/O tasks)
    ->partitions(16)                 // 1-2x concurrency for reduce parallelism
    ->mapperBatchSize(2000)          // Items per worker batch (larger = less overhead)
    ->shuffleChunkSize(50000)        // Shuffle sort memory (larger = fewer merges)
    ->bufferSize(5000)               // Write buffer (larger = fewer syscalls)
    ->workingDirectory('/ssd')       // Use fast storage for large jobs
    ->autoload(__DIR__ . '/vendor/autoload.php')  // Optional: for custom class loading

->mapperBatchSize(200)       // Small batches to reduce serialization
->shuffleChunkSize(2000)     // Small sort chunks
->bufferSize(500)            // Small write buffers

->mapperBatchSize(5000)      // Large batches to reduce process overhead
->shuffleChunkSize(100000)   // Large sort chunks for fast sorting
->bufferSize(10000)          // Large buffers to minimize disk I/O
->workingDirectory('/mnt/fast-ssd/tmp')
bash
composer 
bash
php bin/benchmark-readme.php