PHP code example of wengooooo / blackspider

1. Go to this page and download the library: Download wengooooo/blackspider library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

wengooooo / blackspider example snippets



lackSpider\Downloader\Middleware\RetryMiddleware;
use BlackSpider\Downloader\Middleware\UserAgentMiddleware;
use BlackSpider\Extensions\LoggerExtension;
use BlackSpider\Http\Request;
use BlackSpider\Http\Response;
use BlackSpider\Spider\BasicSpider;
use BlackSpider\Spider\Configuration\Overrides;

class MySpider extends BasicSpider
{
    public function parse(Response $response): \Generator
    {
        /***/
    }

    /** @return Request[] */
    protected function initialRequests(): array
    {
        $yesterday = (new DateTime('yesterday'))->format('Y/m/d');

        return [
            new Request(
                'GET',
                "https://www.httpbin.org/user-agent",
                [$this, 'parse']
            ),
        ];
    }
}

\BlackSpider\BlackSpider::startSpider(
    MySpider::class,
    new Overrides(
        startUrls: ['https://my-override-url.com'],
        downloaderMiddleware: [
            UserAgentMiddleware::class,
            [RetryMiddleware::class, [
                'should_retry_callback' => function (?Response $response = null): bool {
                    if (!$response) {
                        return true;
                    }

                    if(str_contains($response->getBody(), 'user')) {
                        return true;
                    }

                    return false;
                },
                ]
            ],
        ],
        extensions: [
            LoggerExtension::class
        ]
    ),
);