PHP code example of shammaa / laravel-smart-scraper

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

shammaa / laravel-smart-scraper example snippets

namespace App\Scrapers;

use Shammaa\LaravelSmartScraper\Scraper;

class ProductScraper extends Scraper
{
    protected function handle(): array
    {
        $crawler = $this->getCrawler();

        return [
            'title' => $crawler->filter('h1')->text(''),
            'price' => $crawler->filter('.price')->text(''),
            'description' => $crawler->filter('.description')->text(''),
        ];
    }
}

use App\Scrapers\ProductScraper;

$data = ProductScraper::scrape('https://example.com/product/123')
    ->timeout(10000)
    ->run();

dd($data);

use App\Scrapers\ProductScraper;

$data = ProductScraper::scrape('https://example.com/product/123')->run();

$data = ProductScraper::scrape('https://example.com/product/123')
    ->timeout(20000)                    // 20 seconds timeout
    ->proxy('ip:port', 'user', 'pass')  // Use proxy
    ->headers(['Accept-Language' => 'en']) // Custom headers
    ->retry(3, 5)                       // Retry 3 times, wait 5 seconds
    ->cache(false)                      // Disable caching
    ->run();

namespace App\Scrapers;

use Shammaa\LaravelSmartScraper\Scraper;

class ProductScraper extends Scraper
{
    protected function handle(string $selector = 'h1'): array
    {
        $crawler = $this->getCrawler();
        
        return [
            'title' => $crawler->filter($selector)->text(''),
        ];
    }
}

$data = ProductScraper::scrape('https://example.com/product/123')
    ->run(selector: '.product-title');

// Enable caching (default)
$data = ProductScraper::scrape('https://example.com/product/123')
    ->cache(true)
    ->run();

// Disable caching
$data = ProductScraper::scrape('https://example.com/product/123')
    ->cache(false)
    ->run();

'cache' => [
    'ttl' => 3600, // 1 hour
],

// Rate limiting is enabled by default
$data = ProductScraper::scrape('https://example.com/product/123')->run();

// Disable rate limiting
$data = ProductScraper::scrape('https://example.com/product/123')
    ->rateLimit(false)
    ->run();

'rate_limit' => [
    'enabled' => true,
    'max_requests' => 10,    // Max 10 requests
    'per_seconds' => 60,     // Per 60 seconds
],

// Rotation is enabled by default
$data = ProductScraper::scrape('https://example.com/product/123')->run();

'user_agent' => [
    'rotation_enabled' => true,
    'agents' => [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ...',
        // Add more user agents
    ],
],

// Without authentication
$data = ProductScraper::scrape('https://example.com/product/123')
    ->proxy('200.20.14.84:40200')
    ->run();

// With authentication
$data = ProductScraper::scrape('https://example.com/product/123')
    ->proxy('200.20.14.84:40200', 'username', 'password')
    ->run();

// Retry 3 times, wait 1 second between attempts
$data = ProductScraper::scrape('https://example.com/product/123')
    ->retry(3, 1)
    ->run();

'retry' => [
    'enabled' => true,
    'max_attempts' => 3,
    'initial_delay' => 1,        // seconds
    'max_delay' => 60,           // seconds
    'backoff_multiplier' => 2,   // Exponential backoff
    'retryable_status_codes' => [408, 429, 500, 502, 503, 504],
],

// Save screenshot to file
$data = ProductScraper::scrape('https://example.com/product/123')
    ->screenshot(true, storage_path('app/screenshots/product.png'))
    ->run();

// Get screenshot as base64
$data = ProductScraper::scrape('https://example.com/product/123')
    ->screenshot(true)
    ->run();

$screenshotBase64 = $data['screenshot'] ?? null;

// Save PDF to file
$data = ProductScraper::scrape('https://example.com/product/123')
    ->pdf(true, storage_path('app/pdfs/product.pdf'))
    ->run();

// Get PDF as base64
$data = ProductScraper::scrape('https://example.com/product/123')
    ->pdf(true)
    ->run();

$pdfBase64 = $data['pdf'] ?? null;

$data = ProductScraper::scrape('https://example.com/product/123')
    ->headers([
        'Accept-Language' => 'en-US,en;q=0.9',
        'Accept' => 'text/html,application/xhtml+xml',
        'X-Custom-Header' => 'value',
    ])
    ->run();

use Shammaa\LaravelSmartScraper\Services\SchemaValidatorService;

$data = ProductScraper::scrape('https://example.com/product/123')
    ->validate(function ($data) {
        $validator = new SchemaValidatorService();
        
        return $validator->validate($data, [
            'title' => ['

use Shammaa\LaravelSmartScraper\Contracts\MiddlewareInterface;

class CustomHeaderMiddleware implements MiddlewareInterface
{
    public function handle(array $options): array
    {
        $options['headers']['X-Custom'] = 'value';
        return $options;
    }
}

// Use middleware
$data = ProductScraper::scrape('https://example.com/product/123')
    ->middleware(new CustomHeaderMiddleware())
    ->run();

namespace App\Scrapers;

use Shammaa\LaravelSmartScraper\Scraper;

class ProductScraper extends Scraper
{
    protected function handle(): array
    {
        $smart = $this->smart();

        return [
            // Smart extraction - tries multiple selectors automatically
            'title' => $smart->extract('title', [
                'h1',
                '.title',
                '[itemprop="name"]',
                'title',
            ]),

            'price' => $smart->extract('price', [
                '.price',
                '[itemprop="price"]',
                '.amount',
                '.cost',
            ]),

            'image' => $smart->extractAttribute('image', [
                'img.main-image',
                '.product-image img',
                '[itemprop="image"]',
                'img',
            ], 'src'),

            'description' => $smart->extract('description', [
                '.description',
                '[itemprop="description"]',
                '.content',
                'p',
            ]),
        ];
    }
}

$title = $smart->extract('title', [
    'h1.product-title',
    'h1',
    '.title',
    '[itemprop="name"]',
], 'Default Title');

$image = $smart->extractAttribute('image', [
    'img.main-image',
    '.product-image img',
    '[itemprop="image"]',
], 'src', 'default.jpg');

$tags = $smart->extractMultiple('tags', [
    '.tag',
    '.tags a',
    '[itemprop="keywords"]',
], function ($node) {
    return $node->text();
});

'site_profiles' => [
    'amazon' => [
        'url_patterns' => [
            '/amazon\.(com|co\.uk|de|fr|it|es|ca|com\.au)/',
        ],
        'html_patterns' => [
            '#nav-logo' => null,
            '[data-asin]' => null,
        ],
        'selectors' => [
            'title' => [
                '#productTitle',
                'h1.a-size-large',
                'h1',
            ],
            'price' => [
                '.a-price .a-offscreen',
                '#priceblock_dealprice',
                '#priceblock_saleprice',
            ],
        ],
    ],

    'ebay' => [
        'url_patterns' => [
            '/ebay\.(com|co\.uk|de|fr|it|es|ca|com\.au)/',
        ],
        'html_patterns' => [
            '#gh-logo' => null,
            '[data-testid="x-item-title-label"]' => null,
        ],
        'selectors' => [
            'title' => [
                'h1[data-testid="x-item-title-label"]',
                'h1.it-ttl',
                'h1',
            ],
            'price' => [
                '.notranslate',
                '.u-flL.condText',
            ],
        ],
    ],
],

use App\Scrapers\ProductScraper;

// Works with Amazon
$amazonData = ProductScraper::scrape('https://amazon.com/product/123')->run();

// Works with eBay
$ebayData = ProductScraper::scrape('https://ebay.com/itm/123')->run();

// Works with any e-commerce site
$genericData = ProductScraper::scrape('https://example-shop.com/product/123')->run();

protected function handle(): array
{
    $siteType = $this->getSiteType(); // 'amazon', 'ebay', null, etc.
    
    if ($siteType === 'amazon') {
        // Amazon-specific logic
    } elseif ($siteType === 'ebay') {
        // eBay-specific logic
    }
    
    // Or set manually
    $this->setSiteType('custom-site');
    
    return [];
}

// Logs are automatically created
$data = ProductScraper::scrape('https://example.com/product/123')->run();

'monitoring' => [
    'enabled' => true,
    'log_channel' => 'stack',
    'track_metrics' => true,
],

return [
    'cache' => [
        'driver' => 'file',
        'ttl' => 3600,
        'prefix' => 'smart_scraper',
    ],
    
    'rate_limit' => [
        'enabled' => true,
        'max_requests' => 10,
        'per_seconds' => 60,
    ],
    
    'puppeteer' => [
        'node_path' => 'node',
        'script_path' => __DIR__ . '/../resources/js/scraper.js',
        'timeout' => 30000,
        'headless' => true,
    ],
    
    // ... more options
];

->rateLimit(false)

namespace App\Scrapers;

use Shammaa\LaravelSmartScraper\Scraper;

class ProductScraper extends Scraper
{
    protected function handle(): array
    {
        $crawler = $this->getCrawler();

        return [
            'title' => $crawler->filter('h1.product-title')->text(''),
            'price' => $crawler->filter('.price')->text(''),
            'currency' => $crawler->filter('.currency')->text(''),
            'description' => $crawler->filter('.product-description')->text(''),
            'images' => $crawler->filter('.product-images img')->each(function ($node) {
                return $node->attr('src');
            }),
            'rating' => $crawler->filter('.rating')->text(''),
            'reviews_count' => $crawler->filter('.reviews-count')->text(''),
        ];
    }
}

// Usage
$data = ProductScraper::scrape('https://example.com/product/123')
    ->timeout(15000)
    ->retry(3, 2)
    ->run();

namespace App\Scrapers;

use Shammaa\LaravelSmartScraper\Scraper;

class NewsScraper extends Scraper
{
    protected function handle(): array
    {
        $crawler = $this->getCrawler();

        return [
            'title' => $crawler->filter('h1.article-title')->text(''),
            'author' => $crawler->filter('.article-author')->text(''),
            'published_at' => $crawler->filter('.article-date')->attr('datetime'),
            'content' => $crawler->filter('.article-content')->html(),
            'tags' => $crawler->filter('.article-tags a')->each(function ($node) {
                return $node->text();
            }),
            'image' => $crawler->filter('.article-image img')->attr('src'),
        ];
    }
}

// Usage with screenshot
$data = NewsScraper::scrape('https://example.com/news/article-123')
    ->screenshot(true, storage_path('app/screenshots/article.png'))
    ->run();

use App\Scrapers\ProductScraper;
use Shammaa\LaravelSmartScraper\Services\ConcurrentScraperService;

$urls = [
    'https://example.com/product/1',
    'https://example.com/product/2',
    'https://example.com/product/3',
];

$concurrentScraper = new ConcurrentScraperService(maxConcurrent: 5);

$results = $concurrentScraper->scrape($urls, function ($url) {
    return ProductScraper::scrape($url)->run();
});

foreach ($results as $url => $data) {
    echo "Scraped: {$url}\n";
    print_r($data);
}

bash
php artisan vendor:publish --tag=smart-scraper-config

bash
php artisan make:scraper ProductScraper

bash
php artisan make:scraper ProductScraper

bash
php artisan list:scrapers

bash
php artisan make:scraper ProductScraper --smart

[2024-01-01 12:00:00] local.INFO: Scraping started {"url":"https://example.com/product/123",...}
[2024-01-01 12:00:02] local.INFO: Scraping completed {"url":"https://example.com/product/123","duration":"2.5s",...}