PHP code example of webignition / robots-txt-file

1. Go to this page and download the library: Download webignition/robots-txt-file library. Choose the download type require.

2. Extract the ZIP file and open the index.php.

3. Add this code to the index.php. 
    
        
<?php
require_once('vendor/autoload.php');

/* Start to develop here. Best regards https://php-download.com/ */

webignition / robots-txt-file example snippets



use webignition\RobotsTxt\File\Parser;

$parser = new Parser();
$parser->setSource(file_get_contents('http://example.com/robots.txt'));

$robotsTxtFile = $parser->getFile();
 
// Get an array of records
$robotsTxtFile->getRecords();

// Get the list of record-independent directives (such as sitemap directives):
$robotsTxtFile->getNonGroupDirectives()->get();



use webignition\RobotsTxt\File\Parser;
use webignition\RobotsTxt\Inspector\Inspector;

$parser = new Parser();
$parser->setSource(file_get_contents('http://example.com/robots.txt'));

$inspector = new Inspector($parser->getFile());
$inspector->setUserAgent('slurp');

$slurpDirectiveList = $inspector->getDirectives();



use webignition\RobotsTxt\File\Parser;
use webignition\RobotsTxt\Inspector\Inspector;

$parser = new Parser();
$parser->setSource(file_get_contents('http://example.com/robots.txt'));

$inspector = new Inspector($parser->getFile());
$inspector->setUserAgent('slurp');

if ($inspector->isAllowed('/foo')) {
    // Do whatever is needed access to /foo is allowed
}



use webignition\RobotsTxt\File\Parser;

$parser = new Parser();
$parser->setSource(file_get_contents('http://example.com/robots.txt'));

$robotsTxtFile = $parser->getFile();

$sitemapDirectives = $robotsTxtFile->getNonGroupDirectives()->getByField('sitemap');
$sitemapUrl = (string)$sitemapDirectives->first()->getValue();



use webignition\RobotsTxt\File\Parser;
use webignition\RobotsTxt\Inspector\Inspector;

$parser = new Parser();
$parser->setSource(file_get_contents('http://example.com/robots.txt'));

$robotsTxtFile = $parser->getFile();

$inspector = new Inspector($robotsTxtFile);
$inspector->setUserAgent('slurp');

$slurpDisallowDirectiveList = $inspector->getDirectives()->getByField('disallow');