PHP code example of joskfg / laravel-intelligent-scraper

1. Go to this page and download the library: Download joskfg/laravel-intelligent-scraper library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

joskfg / laravel-intelligent-scraper example snippets




use GuzzleHttp\Handler\CurlHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use Goutte\Client as GoutteClient;
use App\MyMiddleware;

$client = new GoutteClient();
$stack = new HandlerStack();

$stack->setHandler(new CurlHandler());
$stack->push(MyMiddleware::getHandler(), 'my_middleware'); // Your custom middleware
$stack->push(Middleware::httpErrors(), 'http_errors'); // Required middleware for the package

$guzzleClient = new GuzzleClient(['handler' => $stack]);
$client->setClient($guzzleClient);


use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;

ScrapedDataset::create([
    'url'  => 'https://test.c/p/my-objective',
    'type' => 'Item-definition-1',
    'data' => [
        [
            'key' => 'title',
            'value' => 'My title',
        ],
        [
            'key' => 'body',
            'value' => 'This is the body content I want to get',
        ],
        [
            'key' => 'images',
            'value' => [
                'https://test.c/images/1.jpg',
                'https://test.c/images/2.jpg',
                'https://test.c/images/3.jpg',
            ],
        ],
    ],
]);


use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;

ScrapedDataset::create([
    'url'     => 'https://test.c/p/my-objective',
    'type'    => 'Item-definition-1',
    'variant' => '8ed10778a83f1266e7ffed90205f7fb61ddcdf78',
    'data'    => [
        [
            'key' => 'title',
            'value' => 'My title',
        ],
        [
            'key' => 'body',
            'value' => 'This is the body content I want to get',
        ],
        [
            'key' => 'images',
            'value' => [
                'https://test.c/images/1.jpg',
                'https://test.c/images/2.jpg',
                'https://test.c/images/3.jpg',
            ],
        ],
    ],
]);


use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;

ScrapedDataset::create([
    'url'  => 'https://test.c/p/my-objective',
    'type' => 'Item-definition-1',
    'variant' => '8ed10778a83f1266e7ffed90205f7fb61ddcdf78',
    'data' => [
        [
            'key' => 'title',
            'value' => 'My title',
        ],
        [
            'key' => 'body',
            'value' => regexp('/^Body starts here, but it is so long that.*$/si'),
        ],
        [
            'key' => 'images',
            'value' => [
                'https://test.c/images/1.jpg',
                'https://test.c/images/2.jpg',
                'https://test.c/images/3.jpg',
            ],
        ],
    ],
]);


use Joskfg\LaravelIntelligentScraper\Scraper\Models\Configuration;

Configuration::create([
    'name'     => 'title',
    'type'     => 'Item-definition-1',
    'xpaths'   => '//*[@id=title]',
    'optional' => false,
    'default'  => [],
]);

Configuration::create([
    'name' => 'category',
    'type' => 'Item-definition-1',
    'xpaths' => ['//*[@id=cat]', '//*[@id=long-cat]'],
    'optional' => true,
    'default'  => [],
]);



scrape('https://test.c/p/my-objective', 'Item-definition-1');



scrape('https://test.c/p/my-objective', 'Item-definition-1', ['id' => 'my-objective']);

/** @var \Joskfg\LaravelIntelligentScraper\Scraper\Events\Scraped $event */
$event->scrapeRequest->url;  // Url scraped
$event->scrapeRequest->type; // Request type
$event->scrapeRequest->context; // Context
$event->scrapedData; // Entity that contains all data scraped and the determined page variant.

/** @var \Joskfg\LaravelIntelligentScraper\Scraper\Events\ScrapeFailed $event */
$event->scrapeRequest->url;  // Url scraped
$event->scrapeRequest->type; // Request type
$event->scrapeRequest->context; // Context

// providers/EventServiceProvider
    protected $listen = [
        Scraped::class => [
            MyListener::class,
        ],
        ScrapeFailed::class => [
            MyListenerFirFailedScrapes::class,
        ],
    ];

return [
    // config/scrapper.php
    'listeners' => [
        'scraped' => [
            'my-type-1' => ListenerForTypeOne::class,
            'my-type-2' => ListenerForTypeTwo::class,
        ],
        'scrape-failed' => [
            'my-type-1' => ListenerFailedForTypeOne::class,
        ],
    ]
];
bash
php artisan vendor:publish --provider="Joskfg\LaravelIntelligentScraper\ScraperProvider" --tag=config
bash
php artisan migrate