1. Go to this page and download the library: Download spiritinlife/php-mapreduce library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
spiritinlife / php-mapreduce example snippets
use Spiritinlife\MapReduce\MapReduceBuilder;
$documents = [
'doc1' => 'the quick brown fox',
'doc2' => 'the lazy dog',
'doc3' => 'quick brown dogs',
];
$results = (new MapReduceBuilder())
->input($documents)
->map(function ($text, $context) {
foreach (str_word_count(strtolower($text), 1) as $word) {
yield [$word, 1];
}
})
->reduce(function ($word, $countsIterator, $context) {
$sum = 0;
foreach ($countsIterator as $count) {
$sum += $count;
}
return $sum;
})
->execute();
// Results are streamed as a generator - iterate to process results
foreach ($results as $word => $count) {
// Output: 'quick' => 2, 'brown' => 2, 'the' => 2, ...
}
// PDO result set
$stmt = $pdo->query('SELECT * FROM large_table');
->input($stmt)
// Generator for memory efficiency
->input((function() use ($pdo) {
$stmt = $pdo->prepare('SELECT * FROM users');
$stmt->execute();
while ($row = $stmt->fetch()) {
yield $row;
}
})())
// Process all files in a directory
->input(new RecursiveIteratorIterator(
new RecursiveDirectoryIterator('/path/to/logs')
))
// Custom file filter
->input((function() {
$dir = new RecursiveDirectoryIterator('/path/to/data');
$iterator = new RecursiveIteratorIterator($dir);
foreach ($iterator as $file) {
if ($file->isFile() && $file->getExtension() === 'log') {
yield $file->getPathname() => file_get_contents($file);
}
}
})())
// Pre-calculated lookup tables or configuration
$userTiers = ['alice' => 'gold', 'bob' => 'silver', 'charlie' => 'bronze'];
$tierMultipliers = ['gold' => 1.5, 'silver' => 1.2, 'bronze' => 1.0];
$purchases = [
['user' => 'alice', 'amount' => 100],
['user' => 'bob', 'amount' => 150],
['user' => 'charlie', 'amount' => 200],
];
$results = (new MapReduceBuilder())
->input($purchases)
// Pass shared data via context (not via 'use' clause!)
->context([
'userTiers' => $userTiers,
'tierMultipliers' => $tierMultipliers,
])
->map(function ($purchase, $context) {
// Access context data in mapper
$tier = $context['userTiers'][$purchase['user']] ?? 'bronze';
$multiplier = $context['tierMultipliers'][$tier];
$bonusPoints = $purchase['amount'] * $multiplier;
yield [$purchase['user'], $bonusPoints];
})
->reduce(function ($user, $pointsIterator, $context) {
// Context also available in reducer
$tier = $context['userTiers'][$user];
// Stream and sum points without loading all into memory
$totalPoints = 0;
foreach ($pointsIterator as $points) {
$totalPoints += $points;
}
return [
'total_points' => $totalPoints,
'tier' => $tier,
];
})
->execute();
foreach ($results as $user => $data) {
echo "{$user} ({$data['tier']}): {$data['total_points']} points\n";
}
// ❌ BROKEN - Variables won't be available in parallel processes
$totalUsers = 1000;
$lookupTable = ['a' => 1, 'b' => 2];
(new MapReduceBuilder())
->input($data)
->map(function ($item, $context) use ($lookupTable) {
// ❌ $lookupTable will be NULL or empty here!
$value = $lookupTable[$item['key']]; // Won't work!
yield [$item['key'], $value];
})
->reduce(function ($key, $valuesIterator, $context) use ($totalUsers) {
// ❌ $totalUsers will be NULL or 0 here!
$sum = 0;
foreach ($valuesIterator as $value) {
$sum += $value;
}
$percentage = $sum / $totalUsers; // Won't work!
return $percentage;
})
->execute();
// ✅ CORRECT - Use context() to pass data
$totalUsers = 1000;
$lookupTable = ['a' => 1, 'b' => 2];
(new MapReduceBuilder())
->input($data)
->context([
'totalUsers' => $totalUsers,
'lookupTable' => $lookupTable,
'config' => ['threshold' => 50]
])
->map(function ($item, $context) {
// ✅ Access via $context parameter
$value = $context['lookupTable'][$item['key']];
yield [$item['key'], $value];
})
->reduce(function ($key, $valuesIterator, $context) {
// ✅ Context available in reducer too
$sum = 0;
foreach ($valuesIterator as $value) {
$sum += $value;
}
$percentage = $sum / $context['totalUsers'];
return $percentage;
})
->execute();
// Phase 1: Calculate counts (stored in memory)
$totalBuyers = 10000;
$eventCounts = [
'follow' => ['seller1' => 500, 'seller2' => 300],
'purchase' => ['seller1' => 200, 'seller2' => 150],
];
// Phase 2: Use counts to calculate scores with MapReduce
$results = (new MapReduceBuilder())
->input($cooccurrences)
->context([
'totalBuyers' => $totalBuyers,
'eventCounts' => $eventCounts,
'minScore' => 5.0
])
->reduce(function ($key, $valuesIterator, $context) {
// Extract shared data from context
$totalBuyers = $context['totalBuyers'];
$eventCounts = $context['eventCounts'];
// Collect values for calculation
$values = iterator_to_array($valuesIterator);
// Use in calculations
$score = calculateScore($values, $totalBuyers, $eventCounts);
if ($score >= $context['minScore']) {
return ['score' => $score, 'count' => count($values)];
}
return null;
})
->execute();
(new MapReduceBuilder())
->concurrent(8) // CPU cores (or 2-3x for I/O tasks)
->partitions(16) // 1-2x concurrency for reduce parallelism
->mapperBatchSize(2000) // Items per worker batch (larger = less overhead)
->shuffleChunkSize(50000) // Shuffle sort memory (larger = fewer merges)
->bufferSize(5000) // Write buffer (larger = fewer syscalls)
->workingDirectory('/ssd') // Use fast storage for large jobs
->autoload(__DIR__ . '/vendor/autoload.php') // Optional: for custom class loading
->mapperBatchSize(200) // Small batches to reduce serialization
->shuffleChunkSize(2000) // Small sort chunks
->bufferSize(500) // Small write buffers
->mapperBatchSize(5000) // Large batches to reduce process overhead
->shuffleChunkSize(100000) // Large sort chunks for fast sorting
->bufferSize(10000) // Large buffers to minimize disk I/O
->workingDirectory('/mnt/fast-ssd/tmp')
bash
composer
bash
php bin/benchmark-readme.php
Loading please wait ...
Before you can download the PHP files, the dependencies should be resolved. This can take some minutes. Please be patient.