PHP code example of datahelm / crawler

1. Go to this page and download the library: Download datahelm/crawler library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php.

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

datahelm / crawler example snippets


'presets' => [
    // ...
    'auctions_pt_BR' => [
        'price_patterns'    => ['/R\$\s*[\d.,]+/'],
        'image_field_hints' => array_merge(
            ['image', 'img', 'photo', 'thumb'],
            ['foto', 'fotos', 'imagem', 'imagens', 'galeria'],
        ),
        'link_field_hints'  => ['url', 'link', 'href', 'permalink', 'lote'],
        'image_path_prefix' => '/arquivos/',
    ],
],


'pipeline_registry' => [
    'trim'            => TrimProcessor::class,
    'absolute_url'    => AbsoluteUrlProcessor::class,
    'schema_coercion' => SchemaCoercionProcessor::class,
],


final class StripEmojiProcessor implements \DataHelm\Crawler\Pipeline\ItemProcessor
{
    public function process(ScrapedItem $item, string $pageUrl): ScrapedItem
    {
        $title = $item->get('title');
        if (is_string($title)) {
            $item->set('title', preg_replace('/[\x{1F300}-\x{1FAFF}]/u', '', $title));
        }

        return $item;
    }
}


// config/crawler.php
'pipeline_registry' => [
    // ...
    'strip_emoji' => \App\Pipeline\StripEmojiProcessor::class,
],


$markdown = (new \DataHelm\Crawler\Markdown\HtmlToMarkdown())->convert($html);

bash
php artisan vendor:publish --tag=crawler-config

bash
php artisan datahelm:scrap:generate "https://example.com/listing" --get-detail=true --robot-name=Example
php artisan datahelm:robot:example --limit=10

bash
php artisan datahelm:scrap:generate "https://en.wikipedia.org/wiki/Web_scraping" \
  --single-page --main-content --robot-name=WikiArticle

bash
php artisan datahelm:scrap:generate <url> --preset=auctions_pt_BR

json
{ "pipeline_names": ["trim", "schema_coercion"] }

bash
php artisan datahelm:scrap:generate <url> --cookie="session_id=abc123; auth_token=xyz789"

bash
php artisan datahelm:scrap:generate <url> --header="Authorization: Bearer eyJhbGciOi..."