PHP code example of tonsoo / php-crawler

1. Go to this page and download the library: Download tonsoo/php-crawler library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

tonsoo / php-crawler example snippets




use Tonsoo\PhpCrawler\Extensions\SitemapExtension;
use Tonsoo\PhpCrawler\Sitemap\SitemapGenerator;
use Tonsoo\PhpCrawler\Sitemap\Writers\RotatingSitemapWriter;

erator(
                writer: new RotatingSitemapWriter(
                    directory: __DIR__ . '/sitemap'
                )
            )
        )
    )
    ->start('https://example.com');


crawler()
    ->displayCrawls(true)
    ->displayMemoryInfo(true)
    ->respectNoIndex(true)
    ->respectNoFollow(true)
    ->respectCanonical(true)
    ->preserveScheme(true)
    ->preserveHost(true)
    ->maxPages(5000)
    ->start('https://example.com');


use Tonsoo\PhpCrawler\Sitemap\SitemapGenerator;
use Tonsoo\PhpCrawler\Sitemap\Writers\XmlSitemapWriter;
use Tonsoo\PhpCrawler\Extensions\SitemapExtension;

crawler()
    ->extension(
        new SitemapExtension(
            generator: new SitemapGenerator(
                writer: new XmlSitemapWriter(
                    path: __DIR__ . '/sitemap/sitemap.xml'
                )
            )
        )
    )
    ->start('https://example.com');


use Tonsoo\PhpCrawler\Sitemap\SitemapGenerator;
use Tonsoo\PhpCrawler\Sitemap\Writers\RotatingSitemapWriter;
use Tonsoo\PhpCrawler\Extensions\SitemapExtension;

crawler()
    ->extension(
        new SitemapExtension(
            generator: new SitemapGenerator(
                writer: new RotatingSitemapWriter(
                    directory: __DIR__ . '/sitemap',
                    baseName: 'sitemap',
                    extension: 'xml',
                    maxUrls: 50000
                )
            )
        )
    )
    ->start('https://example.com');


use Tonsoo\PhpCrawler\Events\OnCrawled;
use Tonsoo\PhpCrawler\Events\OnFinish;
use Tonsoo\PhpCrawler\Events\OnLinkFound;
use Tonsoo\PhpCrawler\Events\OnMismatchContent;
use Tonsoo\PhpCrawler\Events\OnMissingHtmlBody;
use Tonsoo\PhpCrawler\Events\OnStart;

crawler()
    ->onStart(fn (OnStart $event) => print("Starting\n"))
    ->onLinkFound(fn (OnLinkFound $event) => print("{$event->url} -> {$event->link}\n"))
    ->onCrawled(fn (OnCrawled $event) => print("Crawled {$event->page->uri}\n"))
    ->onMissingHtmlBody(fn (OnMissingHtmlBody $event) => print("No HTML: {$event->url}\n"))
    ->onMismatchContent(fn (OnMismatchContent $event) => print("Wrong content type: {$event->url}\n"))
    ->onFinish(fn (OnFinish $event) => print("Done: {$event->totalPages} pages\n"))
    ->start('https://example.com');


use Tonsoo\PhpCrawler\Http\HttpClientInterface;
use Tonsoo\PhpCrawler\Logger\LoggerInterface;
use Tonsoo\PhpCrawler\Analysis\PageAnalyzerInterface;

crawler()
    ->httpClient(new YourHttpClient())
    ->logger(new YourLogger())
    ->pageAnalyzer(new YourAnalyzer())
    ->start('https://example.com');


use Tonsoo\PhpCrawler\Crawler\Exception\LimitExceededException;

try {
    crawler()->maxPages(100)->start('https://example.com');
} catch (LimitExceededException $e) {
    // handle limit reached
}

bash
composer