1. Go to this page and download the library: Download dissnik/robots-txt library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
dissnik / robots-txt example snippets
return [
'cache' => [
// Enable or disable caching for robots.txt responses
'enabled' => env('ROBOTS_TXT_CACHE', true),
// Cache duration in seconds (default: 1 hour)
'duration' => env('ROBOTS_TXT_CACHE_DURATION', 3600),
],
'route' => [
// Automatically register a /robots.txt route
'enabled' => true,
// Middleware to apply to the robots.txt route
'middleware' => ['robots.txt.cache'],
],
// Default environment to use when current environment is not found in environments array
'default_environment' => 'local',
// Environment-specific robots.txt configurations
'environments' => [
'production' => [
// Global directives for production environment
'sitemap' => rtrim(env('APP_URL', 'http://localhost'), '/') . '/sitemap.xml',
// User-agent specific rules for production
'user_agents' => [
// Rules for all user agents (wildcard)
'*' => [
// Paths to disallow access to
'disallow' => [
'/admin',
'/private',
],
// Paths to allow access to (takes precedence over disallow for same paths)
'allow' => [
'/',
],
// Delay between requests in seconds
'crawl-delay' => 1.0,
],
// Rules specific to Googlebot
'Googlebot' => [
'disallow' => ['/private'],
'crawl-delay' => 1.0,
],
],
],
// Local development environment configuration
'local' => [
'user_agents' => [
// Block all access in local environment for safety
'*' => [
'disallow' => ['/'],
],
],
],
],
];
'route' => [
'enabled' => false, // Disable automatic route when using programmatic API
'middleware' => ['robots.txt.cache'],
],
use DissNik\RobotsTxt\Facades\RobotsTxt;
// Define your custom robots.txt route
Route::get('robots.txt', function () {
// Generate robots.txt content programmatically
$content = RobotsTxt::generate();
return response($content, 200, [
'Content-Type' => 'text/plain'
]);
})->name('robots-txt');
use DissNik\RobotsTxt\Facades\RobotsTxt;
// Basic rules - NOTE: Must use callbacks for user-agent specific directives
RobotsTxt::forUserAgent('*', function ($context) {
$context->disallow('/admin')
->allow('/public')
->crawlDelay(1.0);
})->sitemap('https://example.com/sitemap.xml');
// Multiple user agents
RobotsTxt::forUserAgent('Googlebot', function ($context) {
$context->disallow('/private')
->crawlDelay(2.0);
});
RobotsTxt::forUserAgent('Bingbot', function ($context) {
$context->disallow('/secret');
});
// Block all in local development
RobotsTxt::forEnvironment('local', function ($robots) {
$robots->blockAll();
});
// Production rules
RobotsTxt::forEnvironment('production', function ($robots) {
$robots->sitemap('https://example.com/sitemap.xml')
->forUserAgent('*', function ($context) {
$context->allow('/')
->disallow('/admin');
});
});
// Multiple environments
RobotsTxt::forEnvironment(['staging', 'production'], function ($robots) {
$robots->forUserAgent('*', function ($context) {
$context->disallow('/debug');
});
});
// Using when() and unless() methods
RobotsTxt::when($isMaintenanceMode, function ($robots) {
$robots->blockAll();
})->unless($isMaintenanceMode, function ($robots) {
$robots->forUserAgent('*', function ($context) {
$context->allow('/');
});
});
// Block all crawlers
RobotsTxt::blockAll();
// Allow all crawlers
RobotsTxt::allowAll();
// Clear all rules and reload from config
RobotsTxt::reset();
// Clear only programmatic rules
RobotsTxt::clear();
// Clear cache
RobotsTxt::clearCache();
use DissNik\RobotsTxt\Facades\RobotsTxt;
// Get all rules as array
$rules = RobotsTxt::getRules();
// Get all sitemaps
$sitemaps = RobotsTxt::getSitemaps();
// Get all global directives
$directives = RobotsTxt::getDirectives();
// Get directives for specific user agent
$googlebotRules = RobotsTxt::getUserAgentDirectives('Googlebot');
// Check for conflicts (returns array of conflicts)
$conflicts = RobotsTxt::checkConflicts();
// Debug environment rules
$envRules = RobotsTxt::getEnvironmentRules();
// Get all defined user agents
$agents = RobotsTxt::getUserAgents();
// Check if user agent exists
if (RobotsTxt::hasUserAgent('Googlebot')) {
// ...
}
// This will generate only "Allow: /admin" (Allow wins)
RobotsTxt::forUserAgent('*', function ($context) {
$context->disallow('/admin')
->allow('/admin');
});
// Disable caching for current request
config(['robots-txt.cache.enabled' => false]);
// Or use environment variable
putenv('ROBOTS_TXT_CACHE=false');
// Clear cached robots.txt
RobotsTxt::clearCache();
// Custom cache duration (in seconds)
config(['robots-txt.cache.duration' => 7200]); // 2 hours
// Or use environment variable
putenv('ROBOTS_TXT_CACHE_DURATION=7200');
// Disable middleware caching in routes
Route::get('robots.txt', function () {
return response(RobotsTxt::generate(), 200, [
'Content-Type' => 'text/plain'
]);
})->withoutMiddleware('robots.txt.cache');
// Remove global directive
RobotsTxt::removeDirective('sitemap', 'https://example.com/old-sitemap.xml');
// Remove user agent directive
RobotsTxt::removeUserAgentDirective('*', 'disallow', '/admin');
// Remove all sitemaps
RobotsTxt::removeDirective('sitemap');
use DissNik\RobotsTxt\Facades\RobotsTxt;
RobotsTxt::reset()
->forEnvironment('production', function ($robots) {
$robots->sitemap('https://example.com/sitemap.xml')
->sitemap('https://example.com/sitemap-images.xml')
->host('www.example.com')
->forUserAgent('*', function ($context) {
$context->allow('/')
->disallow('/admin')
->disallow('/private')
->disallow('/tmp')
->crawlDelay(1.0);
})
->forUserAgent('Googlebot-Image', function ($context) {
$context->allow('/images')
->crawlDelay(2.0);
});
})
->forEnvironment('local', function ($robots) {
$robots->blockAll();
});
use DissNik\RobotsTxt\Facades\RobotsTxt;
use Illuminate\Support\Facades\Auth;
// Different rules for authenticated users
RobotsTxt::when(Auth::check(), function ($robots) {
$robots->forUserAgent('*', function ($context) {
$context->disallow('/login')
->disallow('/register');
});
})->unless(Auth::check(), function ($robots) {
$robots->forUserAgent('*', function ($context) {
$context->allow('/login')
->allow('/register');
});
});
// Time-based rules
RobotsTxt::when(now()->hour >= 22 || now()->hour < 6, function ($robots) {
$robots->forUserAgent('*', function ($context) {
$context->crawlDelay(5.0); // Slower crawling at night
});
});
// Check generated content
$content = RobotsTxt::generate();
echo $content;
// Debug rules
dd(RobotsTxt::getRules());
// Check all directives
dd(RobotsTxt::getDirectives());
// Check environment detection
dd(app()->environment());
// Check if user agent exists
dd(RobotsTxt::hasUserAgent('Googlebot'));
// Check cache status
dd(config('robots-txt.cache'));
// Check route status
dd(config('robots-txt.route.enabled'));