PHP code example of vdb / php-spider

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

vdb / php-spider example snippets

$spider = new Spider('http://www.dmoz.org');

$spider->addDiscoverer(new XPathExpressionDiscoverer("//div[@id='catalogs']//a"));

$spider->setMaxDepth(1);
$spider->setMaxQueueSize(10);

$statsHandler = new StatsHandler();
$spider->getQueueManager()->getDispatcher()->addSubscriber($statsHandler);
$spider->getDispatcher()->addSubscriber($statsHandler);

$spider->crawl();

echo "\n  ENQUEUED:  " . count($statsHandler->getQueued());
echo "\n  SKIPPED:   " . count($statsHandler->getFiltered());
echo "\n  FAILED:    " . count($statsHandler->getFailed());
echo "\n  PERSISTED:    " . count($statsHandler->getPersisted());

echo "\n\nDOWNLOADED RESOURCES: ";
foreach ($spider->getDownloader()->getPersistenceHandler() as $resource) {
    echo "\n - " . $resource->getCrawler()->filterXpath('//title')->text();
}

use VDB\Spider\Spider;
use VDB\Spider\Discoverer\XPathExpressionDiscoverer;
use VDB\Spider\Filter\Prefetch\AllowedHostsFilter;
use VDB\Spider\PersistenceHandler\FileSerializedResourcePersistenceHandler;
use VDB\Spider\QueueManager\QueueManagerInterface;

$spider = new Spider('https://example.com');

// Configure limits and traversal in one place
$spider
    ->setDownloadLimit(50)                         // Max resources to download
    ->setTraversalAlgorithm(QueueManagerInterface::ALGORITHM_BREADTH_FIRST)
    ->setMaxDepth(2)                               // Max discovery depth
    ->setMaxQueueSize(500)                         // Max URIs in queue
    ->setPersistenceHandler(new FileSerializedResourcePersistenceHandler(__DIR__.'/results'))
    ->addDiscoverer(new XPathExpressionDiscoverer('//a')) // Add discoverers
    ->addFilter(new AllowedHostsFilter(['example.com'])); // Add prefetch filters

// Optional: enable politeness policy (delay between requests to same domain)
$spider->enablePolitenessPolicy(100);

$spider->crawl();

use VDB\Spider\Filter\Prefetch\CachedResourceFilter;
use VDB\Spider\PersistenceHandler\FileSerializedResourcePersistenceHandler;

// Use a fixed spider ID to share cache across runs
$spiderId = 'my-spider-cache';
$spider = new Spider('http://example.com', null, null, null, $spiderId);

// Set up file persistence
$resultsPath = __DIR__ . '/cache';
$spider->getDownloader()->setPersistenceHandler(
    new FileSerializedResourcePersistenceHandler($resultsPath)
);

// Add cache filter - skip resources downloaded within the last hour
$maxAgeSeconds = 3600; // 1 hour (set to 0 to always use cache)
$cacheFilter = new CachedResourceFilter($resultsPath, $spiderId, $maxAgeSeconds);
$spider->getDiscovererSet()->addFilter($cacheFilter);

$spider->crawl();

bash
$ composer