PHP code example of mnb / scraperkit

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

mnb / scraperkit example snippets

declare(strict_types=1);

tions;
use Mnb\ScraperKit\Core\Scraper;

$scraper = new Scraper();

$result = $scraper->crawl(
    'https://example.com',
    new CrawlOptions(
        maxPages: 1,
        maxDepth: 0,
        delayMs: 500,
    )
);

header('Content-Type: application/json');

echo json_encode(
    $result->toArray(false),
    JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
);

declare(strict_types=1);

tions;
use Mnb\ScraperKit\Core\Scraper;

$rules = [
    'title' => 'title',
    'h1' => 'h1',
    'meta_description' => 'meta[name="description"]::attr(content)',
    'links' => 'a::attr(href)[]',
];

$result = (new Scraper())->crawl(
    'https://example.com',
    new CrawlOptions(maxPages: 1, maxDepth: 0),
    $rules
);

header('Content-Type: application/json');

echo json_encode(
    $result->toArray(false),
    JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
);

bash
php vendor/bin/mnb-scraper list

bash
php vendor/bin/mnb-scraper crawl "https://example.com" --max-pages=1 --depth=0 --format=json

bash
php index.php

text
http://localhost/test-mnb-scraperkit/index.php

text
examples/php/index-after-composer-install.php
examples/php/custom-rules-after-composer-install.php

bash
git clone https://github.com/mnagendrababu23/mnb-scraperkit.git
cd mnb-scraperkit
composer install
php bin/mnb-scraper list
php -d zend.assertions=1 -d assert.exception=1 tests/run-tests.php

bash
php bin/mnb-scraper list
php bin/mnb-scraper compat:commands --json
php bin/mnb-scraper release:check . --strict

bash
composer init --name="mnb/test-mnb-scraperkit" --description="Test project for MNB ScraperKit" --type="project" --license="MIT" --

bash
composer init --name="mnb/test-mnb-scraperkit" --description="Test project for MNB ScraperKit" --type="project" --license="MIT" --

bash
php bin/mnb-scraper crawl "https://example.com" --max-pages=1 --depth=0 --format=json

bash
php bin/mnb-scraper crawl "https://example.com" --max-pages=5 --depth=1 --delay-ms=1000 --format=json

bash
php bin/mnb-scraper pipeline:run storage/jobs/example/crawl.json --output=storage/jobs/example/pipeline --format=both

bash
php bin/mnb-scraper publisher:list --json
php bin/mnb-scraper publisher:show elsevier
php bin/mnb-scraper publisher:seeds --format=txt --output=storage/publisher-seeds.txt
php bin/mnb-scraper publisher:plan --max-pages=10 --delay-ms=3500 --output=storage/publisher-crawl-jobs.json
php bin/mnb-scraper publisher:schema --format=csv --output=storage/article-schema.csv
php bin/mnb-scraper publisher:normalize storage/source-records.json --publisher=elsevier --output=storage/article-metadata.json

bash
php bin/mnb-scraper user:create [email protected] --display-name="Admin User" --role=owner
php bin/mnb-scraper workspace:create seo-team [email protected] --profile=seo
php bin/mnb-scraper workspace:assign-user seo-team [email protected] --role=analyst
php bin/mnb-scraper enterprise:doctor
php bin/mnb-scraper audit:events --limit=20

bash
php bin/mnb-scraper security:audit --format=html --output=storage/security-audit.html
php bin/mnb-scraper security:doctor
php bin/mnb-scraper security:secrets-scan
php bin/mnb-scraper compliance:report --format=html --output=storage/compliance-report.html

bash
php bin/mnb-scraper distributed:doctor --distributed-adapter=auto --json
php bin/mnb-scraper distributed:enqueue --command=crawl --arg=https://example.com --pipeline --distributed-adapter=file
php bin/mnb-scraper worker:distributed --distributed-adapter=file --stop-when-empty --max-jobs=5

bash
set MNB_SCRAPERKIT_REDIS_URL=redis://127.0.0.1:6379/0
php bin/mnb-scraper distributed:enqueue --command=bulk:crawl --arg=urls.txt --distributed-adapter=redis
php bin/mnb-scraper worker:distributed --distributed-adapter=redis --worker-group=seo-workers --max-jobs=100

bash
php bin/mnb-scraper export:connector-list
php bin/mnb-scraper export:connector-validate
php bin/mnb-scraper export:manifest storage/jobs/example --output=storage/jobs/example/export-manifest.json
php bin/mnb-scraper export:deliver local_exports --dir=storage/jobs/example
php bin/mnb-scraper export:connector-test webhook_dry_run

bash
php bin/mnb-scraper browser:session-create client_portal --domain=example.com --login-url=https://example.com/login
php bin/mnb-scraper browser:login client_portal --url=https://example.com/login
php bin/mnb-scraper browser:session-test client_portal https://example.com/dashboard --browser=always --render --json
php bin/mnb-scraper crawl https://example.com/dashboard --browser=auto --session=client_portal --pipeline

bash
php bin/mnb-scraper api:routes
php bin/mnb-scraper api:routes --json

bash
php bin/mnb-scraper api:token
php bin/mnb-scraper api:token --prefix=mnb_sk

bash
export MNB_SCRAPERKIT_API_TOKEN="paste-generated-token-here"
php bin/mnb-scraper api:serve --host=127.0.0.1 --port=8787

bash
php bin/mnb-scraper webhook:test --event=job.completed

bash
php bin/mnb-scraper plugin:list
php bin/mnb-scraper plugin:list --json

bash
php bin/mnb-scraper plugin:show mnb.example.profile-addon

bash
php bin/mnb-scraper plugin:validate plugins/example-profile-addon
php bin/mnb-scraper plugin:doctor

bash
php bin/mnb-scraper profile:list
php bin/mnb-scraper profile:show research-paper
php bin/mnb-scraper extract:rules https://example.com/article --profile=research-paper

bash
php bin/mnb-scraper profile:list

bash
php bin/mnb-scraper profile:show ecommerce
php bin/mnb-scraper profile:show ecommerce --json

bash
php bin/mnb-scraper profile:validate config/profiles/ecommerce.json

bash
php bin/mnb-scraper extract:rules https://example.com/product-page --profile=ecommerce

bash
php bin/mnb-scraper crawl https://example.com/product-page --profile=ecommerce --pipeline --job-dir=storage/jobs/product-test
php bin/mnb-scraper pipeline:run storage/jobs/product-test/crawl.json --profile=ecommerce --output-dir=storage/jobs/product-test/pipeline

bash
php bin/mnb-scraper export:records storage/jobs/job-id/pipeline/records.json --format=csv
php bin/mnb-scraper export:records storage/jobs/job-id/pipeline/records.json --format=xml
php bin/mnb-scraper export:records storage/jobs/job-id/pipeline/records.json --format=html

bash
php bin/mnb-scraper export:failed storage/jobs/job-id/crawl.json --format=csv --

bash
php bin/mnb-scraper export:validation storage/jobs/job-id/pipeline/records.json --format=csv

bash
php bin/mnb-scraper report:summary storage/jobs/job-id --format=html
php bin/mnb-scraper report:summary storage/jobs/job-id --format=json

bash
php bin/mnb-scraper bundle:create storage/jobs/job-id

text
examples/crawl/sample-urls.txt
examples/crawl/windows-crawl-presets.json
examples/crawl/windows-crawl-notes.txt

bash
php bin/mnb-scraper list
php bin/mnb-scraper crawl --help
php bin/mnb-scraper pipeline:run --help

bash
php bin/mnb-scraper retry:plan storage/jobs/example/crawl.json --output=storage/jobs/example/retry-plan.json
php bin/mnb-scraper retry:plan --failed-jobs --json

bash
php bin/mnb-scraper queue:retry-safe
php bin/mnb-scraper queue:retry-safe --dry-run --json

bash
php bin/mnb-scraper schedule:create --command=crawl https://example.com --every-minutes=60 --profile=seo
php bin/mnb-scraper schedule:list
php bin/mnb-scraper schedule:run-due

bash
php bin/mnb-scraper schedule:run-due
php bin/mnb-scraper worker:run --stop-when-empty

bash
php bin/mnb-scraper monitor:summary
php bin/mnb-scraper monitor:summary --json
php bin/mnb-scraper monitor:stale-locks --ttl-seconds=900

bash
php bin/mnb-scraper db:init

bash
php bin/mnb-scraper db:init --sqlite=storage/database/project.sqlite
php bin/mnb-scraper db:status --sqlite=storage/database/project.sqlite

bash
php bin/mnb-scraper db:save-crawl storage/jobs/example/crawl.json --job-id=example
php bin/mnb-scraper db:save-pipeline storage/jobs/example/pipeline/records.json --job-id=example

bash
php bin/mnb-scraper db:export mnb_storage_records --format=csv --output=records-from-db.csv

bash
php bin/mnb-scraper db:init --database-url="mysql:host=127.0.0.1;dbname=mnb_scraperkit;charset=utf8mb4" --db-user=root --db-pass=secret

bash
php bin/mnb-scraper source:sitemap "https://example.com/sitemap.xml" --format=txt --output=urls.txt
php bin/mnb-scraper bulk:crawl urls.txt --delay-ms=1000 --pipeline

bash
php bin/mnb-scraper source:csv urls.csv --url-column=url --format=txt --output=urls.txt

bash
php bin/mnb-scraper source:json urls.json --path="items.*.url" --format=txt --output=urls.txt

bash
php bin/mnb-scraper browser:test https://example.com --browser=auto --json

bash
php bin/mnb-scraper crawl https://example.com --browser=auto --pipeline