PHP code example of jez500 / web-scraper-for-laravel

1. Go to this page and download the library: Download jez500/web-scraper-for-laravel library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

jez500 / web-scraper-for-laravel example snippets


use Jez500\WebScraperForLaravel\Facades\WebScraper;

$scraper = WebScraper::http()->from('https://example.com')->get();
$title = $scraper->getSelector('title')->first();

$data = WebScraper::http()
    ->from('https://example.com')
    ->get()
    ->getJson('user.email')
    ->first();

use Jez500\WebScraperForLaravel\Facades\WebScraper;

// Get an instance of the scraper with the body of the page loaded
$scraper = WebScraper::http()->from('https://example.com')->get();

// Get the full page body
$body = $scraper->getBody();

// Get the first title element
$title = $scraper->getSelector('title')->first(); 

// Get the content attribute of the first meta tag with property og:image
$image = $scraper->getSelector('meta[property=og:image]|content')->first(); 

// Get all paragraph innerHtml as an array
$paragraphs = $scraper->getSelector('p')->all();

// Get all links as an array
$links = $scraper->getSelector('a|href')->all();

// Get the first matching element with a specific class
$price = $scraper->getSelector('.product-price')->first();

// Get the first h1 element
$h1 = $scraper->getXpath('//h1')->first();

// Get the href attribute of the first link
$linkHref = $scraper->getXpath('//a', 'attr', ['href'])->first();

// Get multiple elements with a specific attribute
$images = $scraper->getXpath('//img[@class="product-image"]')->all();

// Complex XPath query
$productTitle = $scraper->getXpath('//div[@class="product"]/h1')->first();

// Extract user from JSON-like string
$author = $scraper->getRegex('~"user"\:"(.*)"~')->first();

// Extract email addresses
$emails = $scraper->getRegex('~[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}~')->all();

// Extract phone numbers
$phones = $scraper->getRegex('~\b\d{3}[-.]?\d{3}[-.]?\d{4}\b~')->all();

// Get nested JSON data from API response
$author = WebScraper::http()
    ->from('https://api.example.com/page.json')
    ->get()
    ->getJson('user.name')
    ->first();

// Extract all items from an array
$products = WebScraper::http()
    ->from('https://api.example.com/products.json')
    ->get()
    ->getJson('products.*.name')
    ->all();

// Extract multiple nested fields
$emails = WebScraper::http()
    ->from('https://api.example.com/users.json')
    ->get()
    ->getJson('users.*.email')
    ->all();

// Get title from a JavaScript-rendered page
$title = WebScraper::api()
    ->from('https://example.com')
    ->get()
    ->getSelector('title')
    ->first();

// Extract data from a Single Page Application
$data = WebScraper::api()
    ->from('https://example.com/app')
    ->get()
    ->getSelector('[data-content]')
    ->first();

// Extract article content
$content = WebScraper::extract()
    ->from('https://example.com/article')
    ->get()
    ->getBody();

// With custom headers
$content = WebScraper::extract()
    ->from('https://example.com/article')
    ->setOptions(['headers' => ['Accept-Language' => 'en-US']])
    ->get()
    ->getBody();

// Use custom Extract API endpoint
$content = WebScraper::extract()
    ->setExtractApiBaseUrl('http://my-extract-service:3000/parser')
    ->from('https://example.com/article')
    ->get()
    ->getBody();

use Illuminate\Support\Facades\Http;
use Jez500\WebScraperForLaravel\AbstractWebScraper;
use Jez500\WebScraperForLaravel\Drivers\WebScraperDriverInterface;
use Jez500\WebScraperForLaravel\Facades\WebScraper;

WebScraper::extend('residential-proxy', function () {
    return new class implements WebScraperDriverInterface {
        public function fetch(AbstractWebScraper $scraper): string
        {
            $response = Http::withToken(config('services.proxy.token'))
                ->post(config('services.proxy.endpoint'), [
                    'url' => $scraper->getUrl(),
                    'country' => 'us',
                ]);

            return data_get($response->json(), 'html', '');
        }
    };
});

$scraper = WebScraper::driver('residential-proxy')
    ->from('https://example.com')
    ->get();

$title = $scraper->getSelector('title')->first();

use Jez500\WebScraperForLaravel\Dto\FieldExtractionDto;
use Jez500\WebScraperForLaravel\Dto\ScrapeSchemaDto;

// Define a schema for structured data extraction
$schema = ScrapeSchemaDto::fromArray([
    'fields' => [
        'title' => [
            'type' => 'css',
            'value' => 'title',
        ],
        'description' => [
            'type' => 'css',
            'value' => 'meta[name=description]|content',
        ],
        'image' => [
            'type' => 'xpath',
            'value' => '//meta[@property="og:image"]/@content',
        ],
    ],
]);

// Apply schema and get structured data
$data = WebScraper::http()
    ->from('https://example.com')
    ->get()
    ->fromDto($schema);

// $data will be: ['title' => '...', 'description' => '...', 'image' => '...']

$title = FieldExtractionDto::fromArray([
    'type' => 'css',
    'value' => 'title',
]);

$data = WebScraper::http()
    ->from('https://example.com')
    ->get()
    ->fromDto($title);

// $data will be: ['field' => 'Example Domain']

// Set a custom user agent
$scraper = WebScraper::http()
    ->from('https://example.com')
    ->withUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
    ->get();

// Cache response for 1 hour
$scraper = WebScraper::http()
    ->from('https://example.com')
    ->cacheFor(3600)  // seconds
    ->get();

try {
    $scraper = WebScraper::http()
        ->from('https://example.com')
        ->get();

    $title = $scraper->getSelector('title')->first();
} catch (\Exception $e) {
    // Handle scraping errors
    Log::error('Scraping failed', ['error' => $e->getMessage()]);
}

use Illuminate\Support\Facades\Http;
use Jez500\WebScraperForLaravel\Facades\WebScraper;

public function test_scraping_basic_page()
{
    // Mock the HTTP response
    Http::fake([
        'example.com/*' => Http::response('<html><title>Test Page</title></html>', 200),
    ]);

    $scraper = WebScraper::http()->from('https://example.com')->get();
    $title = $scraper->getSelector('title')->first();

    $this->assertEquals('Test Page', $title);
}