PHP code example of changhorizon / crawler

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

changhorizon / crawler example snippets

use ChangHorizon\Crawler\Crawler;
use ChangHorizon\Crawler\Http\CurlHttpClient;
use ChangHorizon\Crawler\Storage\PdoDocumentStorage;
use Psr\Log\NullLogger;

$startUrl = 'https://example.com';

$httpClient = new CurlHttpClient();
$storage = new PdoDocumentStorage($pdoConnection); // $pdoConnection 为已初始化的 PDO 实例
$logger = new NullLogger(); // 可替换为任何 PSR-3 实现，如 Monolog

$crawler = new Crawler($startUrl, $httpClient, $storage, $logger);
$crawler->run();

$crawlDelay = 3;

$crawler = new Crawler($startUrl, $httpClient, $storage, $logger, $crawlDelay);
$crawler->run();

public function __construct(
    string $startUrl,
    HttpClientInterface $httpClient,
    DocumentStorageInterface $storage,
    LoggerInterface $logger,
    int $crawlDelay = 1,
);

public function run(): void

private function fetchUrlContent(string $url): ?string

private function scrapePage(string $url): void

private function extractLinks(string $html): array

private function resolveUrl(string $href): ?string

interface DocumentStorageInterface
{
    /**
     * 保存或更新文档内容。
     *
     * @param string $url
     * @param Document $document
     * @return bool 返回 true 表示内容发生变化已保存
     */
    public function save(string $url, Document $document): bool;

    /**
     * 检查指定 URL 是否已处理过。
     *
     * @param string $url
     * @return bool
     */
    public function isProcessed(string $url): bool;
}

interface HttpClientInterface
{
    /**
     * 发送 GET 请求。
     *
     * @param string $url
     * @return array|null 包含 statusCode, contentType 和 content，若失败则为 null
     */
    public function get(string $url): ?array;
}

txt
src
├── Crawler.php
├── Document.php
├── Http
│   ├── CurlHttpClient.php
│   └── HttpClientInterface.php
└── Storage
    ├── DocumentStorageInterface.php
    └── PdoDocumentStorage.php