PHP code example of j0k3r / graby

    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

    

j0k3r / graby example snippets

$article = 'http://www.bbc.com/news/entertainment-arts-32547474';
$request = 'http://example.org/full-text-rss/makefulltextfeed.php?format=json&url='.urlencode($article);
$result  = @file_get_contents($request);

use Graby\Graby;

$article = 'http://www.bbc.com/news/entertainment-arts-32547474';

$graby = new Graby();
$result = $graby->fetchContent($article);

var_dump($result->getResponse()->getStatus()); // 200
var_dump($result->getHtml()); // "[Fetched and readable content…]"
var_dump($result->getTitle()); // "Ben E King: R&B legend dies at 76"
var_dump($result->getLanguage()); // "en-GB"
var_dump($result->getDate()); // "2015-05-01T16:24:37+01:00"
var_dump($result->getAuthors()); // ["BBC News"]
var_dump((string) $result->getResponse()->getEffectiveUri()); // "http://www.bbc.com/news/entertainment-arts-32547474"
var_dump($result->getImage()); // "https://ichef-1.bbci.co.uk/news/720/media/images/82709000/jpg/_82709878_146366806.jpg"
var_dump($result->getSummary()); // "Ben E King received an award from the Songwriters Hall of Fame in …"
var_dump($result->getIsNativeAd()); // false
var_dump($result->getResponse()->getHeaders()); /*
[
  'server' => ['Apache'],
  'content-type' => ['text/html; charset=utf-8'],
  'x-news-data-centre' => ['cwwtf'],
  'content-language' => ['en'],
  'x-pal-host' => ['pal074.back.live.cwwtf.local:80'],
  'x-news-cache-id' => ['13648'],
  'content-length' => ['157341'],
  'date' => ['Sat, 29 Apr 2017 07:35:39 GMT'],
  'connection' => ['keep-alive'],
  'cache-control' => ['private, max-age=60, stale-while-revalidate'],
  'x-cache-action' => ['MISS'],
  'x-cache-age' => ['0'],
  'x-lb-nocache' => ['true'],
  'vary' => ['X-CDN,X-BBC-Edge-Cache,Accept-Encoding'],
]
*/

var_dump($result->getResponse()->getStatus()); // 200
var_dump($result->getHtml()); // "[unable to retrieve full-text content]"
var_dump($result->getTitle()); // "BBC - 404: Not Found"
var_dump($result->getLanguage()); // "en-GB"
var_dump($result->getDate()); // null
var_dump($result->getAuthors()); // []
var_dump((string) $result->getResponse()->getEffectiveUri()); // "http://www.bbc.co.uk/404"
var_dump($result->getImage()); // null
var_dump($result->getSummary()); // "[unable to retrieve full-text content]"
var_dump($result->getIsNativeAd()); // false
var_dump($result->getResponse()->getHeaders()); // […]

use Graby\Graby;

$article = 'http://www.bbc.com/news/entertainment-arts-32547474';
// use your own way to retrieve html or to provide html
$html = ...

$graby = new Graby();
$result = $graby->cleanupHtml($html, $article);

$logs = $this->get('monolog.handler.graby')->getRecords();

    use Graby\Graby;
    use GuzzleHttp\Client as GuzzleClient;
    use Http\Adapter\Guzzle7\Client as GuzzleAdapter;

    $guzzle = new GuzzleClient([
        'timeout' => 2,
    ]);
    $graby = new Graby(client: new GuzzleAdapter($guzzle));
    

use Graby\Config\ContentLinks;
use Graby\Config\ContentTypeAction;
use Graby\Config\LogLevel;
use Graby\Extractor\ContentExtractorConfig;
use Graby\Extractor\HttpClientConfig;
use Graby\Extractor\ReadabilityConfig;
use Graby\Graby;
use Graby\GrabyConfig;
use Graby\SiteConfig\ConfigBuilderConfig;

$graby = new Graby(new GrabyConfig(
    // Enable or disable debugging.
    // This will only generate log information in a file (log/graby.log)
    debug: false,
    // use LogLevel::Debug value if you want more data (HTML at each step for example) to be dumped in a different file (log/html.log)
    logLevel: LogLevel::Info,
    // If enabled relative URLs found in the extracted content are automatically rewritten as absolute URLs.
    rewriteRelativeUrls: true,
    // If enabled, we will try to follow single page links (e.g. print view) on multi-page articles.
    // Currently this only happens for sites where single_page_link has been defined
    // in a site config file.
    singlepage: true,
    // If enabled, we will try to follow next page links on multi-page articles.
    // Currently this only happens for sites where next_page_link has been defined
    // in a site config file.
    multipage: true,
    // Error message when content extraction fails
    errorMessage: '[unable to retrieve full-text content]',
    // Default title when we won't be able to extract a title
    errorMessageTitle: 'No title found',
    // List of URLs (or parts of a URL) which will be accept.
    // If the list is empty, all URLs (except those specified in the blocked list below)
    // will be permitted.
    // Example: array('example.com', 'anothersite.org');
    allowedUrls: [],
    // List of URLs (or parts of a URL) which will be not accept.
    // Note: this list is ignored if allowed_urls is not empty
    blockedUrls: [],
    // If enabled, we'll pass retrieved HTML content through htmLawed with
    // safe flag on and style attributes denied, see
    // http://www.bioinformatics.org/phplabware/internal_utilities/htmLawed/htmLawed_README.htm#s3.6
    // Note: if enabled this will also remove certain elements you may want to preserve, such as iframes.
    xssFilter: true,
    // Here you can define different actions based on the Content-Type header returned by server.
    // MIME type as key, action as value.
    // Valid actions:
    // * ContentTypeAction::Exclude - exclude this item from the result
    // * ContentTypeAction::Link - create HTML link to the item
    contentTypeExc: [
       'application/zip' => ['action' => ContentTypeAction::Link, 'name' => 'ZIP'],
       'application/pdf' => ['action' => ContentTypeAction::Link, 'name' => 'PDF'],
       'image' => ['action' => ContentTypeAction::Link, 'name' => 'Image'],
       'audio' => ['action' => ContentTypeAction::Link, 'name' => 'Audio'],
       'video' => ['action' => ContentTypeAction::Link, 'name' => 'Video'],
       'text/plain' => ['action' => ContentTypeAction::Link, 'name' => 'Plain text'],
    ],
    // How we handle link in content
    // Valid values :
    // * ContentLinks::Preserve: nothing is done
    // * ContentLinks::Footnotes: convert links as footnotes
    // * ContentLinks::Remove: remove all links
    contentLinks: ContentLinks::Preserve,
    httpClient: new HttpClientConfig(
        // User-Agent used to fetch content
        uaBrowser: 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.92 Safari/535.2',
        // default referer when fetching content
        defaultReferer: 'http://www.google.co.uk/url?sa=t&source=web&cd=1',
        // Currently allows simple string replace of URLs.
        // Useful for rewriting certain URLs to point to a single page or HTML view.
        // Although using the single_page_link site config instruction is the preferred way to do this, sometimes, as
        // with Google Docs URLs, it's not possible.
        rewriteUrl: [
            'docs.google.com' => ['/Doc?' => '/View?'],
            'tnr.com' => ['tnr.com/article/' => 'tnr.com/print/article/'],
            '.m.wikipedia.org' => ['.m.wikipedia.org' => '.wikipedia.org'],
            'm.vanityfair.com' => ['m.vanityfair.com' => 'www.vanityfair.com'],
        ],
        // Prevent certain file/mime types
        // HTTP responses which match these content types will
        // be returned without body.
        headerOnlyTypes: [
           'image',
           'audio',
           'video',
        ],
        // URLs ending with one of these extensions will
        // prompt Humble HTTP Agent to send a HEAD request first
        // to see if returned content type matches $headerOnlyTypes.
        headerOnlyClues: ['mp3', 'zip', 'exe', 'gif', 'gzip', 'gz', 'jpeg', 'jpg', 'mpg', 'mpeg', 'png', 'ppt', 'mov'],
        // User Agent strings - mapping domain names
        userAgents: [],
        // AJAX triggers to search for.
        // for AJAX sites, e.g. Blogger with its dynamic views templates.
        ajaxTriggers: [
            "<meta name='fragment' content='!'",
            '<meta name="fragment" content="!"',
            "<meta content='!' name='fragment'",
            '<meta content="!" name="fragment"',
        ],
        // number of redirection allowed until we assume request won't be complete
        maxRedirect: 10,
    ),
    extractor: new ContentExtractorConfig(
        defaultParser: \Graby\Extractor\Parser::Libxml,
        // key is fingerprint (fragment to find in HTML)
        // value is host name to use for site config lookup if fingerprint matches
        // \s* match anything INCLUDING new lines
        fingerprints: [
            '/\<meta\s*content=([\'"])blogger([\'"])\s*name=([\'"])generator([\'"])/i' => 'fingerprint.blogspot.com',
            '/\<meta\s*name=([\'"])generator([\'"])\s*content=([\'"])Blogger([\'"])/i' => 'fingerprint.blogspot.com',
            '/\<meta\s*name=([\'"])generator([\'"])\s*content=([\'"])WordPress/i' => 'fingerprint.wordpress.com',
        ],
        configBuilder: new ConfigBuilderConfig(
            // Directory path to the site config folder WITHOUT trailing slash
            siteConfig: [],
            hostnameRegex: '/^(([a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9-]*[A-Za-z0-9])$/',
        ),
        readability: new ReadabilityConfig(
            // filters might be like array('regex' => 'replace with')
            // for example, to remove script content: array('!<script[^>]*>(.*?)</script>!is' => '')
            preFilters: [],
            postFilters: [],
        ),
        srcLazyLoadAttributes: [
            'data-src',
            'data-lazy-src',
            'data-original',
            'data-sources',
            'data-hi-res-src',
        ],
        // these JSON-LD types will be ignored
        jsonLdIgnoreTypes: ['Organization', 'WebSite', 'Person', 'VideoGame'],
    ),
));

 php
use Graby\Graby;

$article = 'http://www.bbc.com/news/entertainment-arts-32547474';

$input = '<html>[...]</html>';

$graby = new Graby();
$graby->setContentAsPrefetched($input);
$result = $graby->fetchContent($article);