1. Go to this page and download the library: Download blogdaren/phpcreeper library. Choose the download type require.
2. Extract the ZIP file and open the index.php.
3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
blogdaren / phpcreeper example snippets
//Note: if this example provided by the PHPCreeper does not work as expected,
//check to modify the crawl rules, as the source DOM is most likely updated.
er\Timer;
//switch runtime language between `zh` and `en`, default is `zh`【version >= 1.3.7】
PHPCreeper::setLang('en');
//enable the single worker mode so that we can run without redis, however, you should note
//it will be limited to run only all the downloader workers in this case【version >= 1.3.2】
//PHPCreeper::enableMultiWorkerMode(false);
//set master pid file manually as needed【version >= 1.3.8】
//PHPCreeper::setMasterPidFile('/path/to/master.pid');
//set worker log file when start as daemon mode as needed【version >= 1.3.8】
//PHPCreeper::setLogFile('/path/to/phpcreeper.log');
//note that `predis` will be the default redis client since【version >= 1.4.2】
//but you could still switch it to be `redis` if you prefer to use ext-redis
//PHPCreeper::setDefaultRedisClient('redis');
//set default timezone, default is `Asia/Shanghai`【version >= 1.5.4】
//PHPCreeper::setDefaultTimezone('Asia/Shanghai');
//redirect all stdandard out to file when run as daemonize【version >= 1.7.0】
//PHPCreeper::setStdoutFile("/path/to/stdout.log");
//set default headless browser, default is `chrome`【version >= 1.8.7】
//PHPCreeper::setDefaultHeadlessBrowser('chrome');
//if the child process don't exit within timeout, then force to kill it【version >= 2.0.0】
//PHPCreeper::setChildProcessStopTimeout(5);
//Global-Redis-Config: support array value with One-Dimension or Two-Dimension,
//NOTE: since v1.6.4, it's been upgraded to use a more secure and officially
//recommended distributed red lock mechanism by default, but it will use the
//old version of the lock mechanism degenerate only when all the redis instances
//are explicitly configured with the option [use_red_lock === false] as below.
//for details on how to configure the value, refer to the Follow-Up sections.
$config['redis'] = [
[
'host' => '127.0.0.1',
'port' => 6379,
'database' => '0',
'auth' => false,
'pass' => 'guest',
'prefix' => 'PHPCreeper',
'connection_timeout' => 5,
'read_write_timeout' => 0,
'use_red_lock' => true,
],
];
//Global-Task-Config: the context member configured here is a global context,
//we can also set a private context for each task, finally the global context
//and private task context will adopt the strategy of merging and covering.
//you can free to customize various context settings, including user-defined,
//for details on how to configure it, please refer to the Follow-Up sections.
$config['task'] = [
'context' => [
'cache_enabled' => true,
'cache_directory' => sys_get_temp_dir() . '/DownloadCache4PHPCreeper/',
'allow_url_repeat' => true,
'headless_browser' => ['headless' => false],
],
];
function startAppProducer()
{
global $config;
$producer = new Producer($config);
$producer->setName('AppProducer')->setCount(1);
$producer->onProducerStart = function($producer){
//private task context which will be merged with global context
$private_task_context = [];
//use 1D-array:we can use either `createTask()` or `createMultiTask()` API
$task = array(
'url' => "https://forecast.weather.gov/MapClick.php?lat=47.4113&lon=-120.5563",
'rule' => [
'period' => ['#seven-day-forecast-container ul li p.period-name', 'text'],
'weather' => ['#seven-day-forecast-container ul li p.short-desc', 'text'],
'temperature' => ['#seven-day-forecast-container ul li p.temp', 'text'],
],
'rule_name' => '', //md5($task_id) will be the rule_name if leave empty
'refer' => '',
'type' => 'text', //it has lost the original concept setting, feel free to use
'method' => 'get',
'parser' => '', //router to one random target parser if leave empty [ip:port]
"context" => $private_task_context,
);
$producer->createTask($task);
//use 2D-array: since it is multitasking, we can only use the `createMultiTask()` API
$task = array(
array(
'url' => "https://forecast.weather.gov/MapClick.php?lat=47.4113&lon=-120.5563",
'rule' => [
'period' => ['#seven-day-forecast-container ul li p.period-name', 'text'],
'weather' => ['#seven-day-forecast-container ul li p.short-desc', 'text'],
'temperature' => ['#seven-day-forecast-container ul li p.temp', 'text'],
],
'rule_name' => 'r1',
"context" => $private_task_context,
),
array(
'url' => "https://forecast.weather.gov/MapClick.php?lat=47.4113&lon=-120.5563",
'rule' => [
'period' => ['#seven-day-forecast-container ul li p.period-name', 'text'],
'weather' => ['#seven-day-forecast-container ul li p.short-desc', 'text'],
'temperature' => ['#seven-day-forecast-container ul li p.temp', 'text'],
],
'rule_name' => 'r2',
"context" => $private_task_context,
),
);
$producer->createMultiTask($task);
//use headless browser to crawl dynamic page rendered by javascript
$private_task_context['headless_browser']['headless'] = true;
$dynamic_task = array(
'url' => 'https://www.toutiao.com',
'rule' => array(
'title' => ['div.show-monitor ol li a', 'aria-label'],
'link' => ['div.show-monitor ol li a', 'href'],
),
'context' => $private_task_context,
);
$producer->createTask($dynamic_task);
};
}
function startAppDownloader()
{
global $config;
$downloader = new Downloader($config);
//set the client socket address based on the listening parser server
$downloader->setName('AppDownloader')->setCount(2)->setClientSocketAddress([
'ws://127.0.0.1:8888',
]);
$downloader->onDownloadBefore = function($downloader, $task){
//disable http ssl verify in any of the following two ways
//$downloader->httpClient->disableSSL();
//$downloader->httpClient->setOptions(['verify' => false]);
};
//more downloader or download callbacks frequently used
//$downloader->onDownloaderStart = function($downloader){};
//$downloader->onDownloaderStop = function($downloader){};
//$downloader->onDownloaderMessage = function($downloader, $parser_reply){};
//$downloader->onDownloaderConnectToParser = function($connection){};
//$downloader->onDownloadStart = function($downloader, $task){};
//$downloader->onDownloadAfter = function($downloader, $download_data, $task){};
//$downloader->onDownloadFail = function($downloader, $error, $task){};
//$downloader->onDownloadTaskEmpty = function($downloader){};
//$downloader->onHeadlessBrowserOpenPage = function($downloader, $browser, $page, $url){};
}
function startAppParser()
{
$parser = new Parser();
$parser->setName('AppParser')->setCount(1)->setServerSocketAddress('websocket://0.0.0.0:8888');
$parser->onParserExtractField = function($parser, $download_data, $fields){
pprint($fields);
};
//more parser callbacks frequently used
//$parser->onParserStart = function($parser){};
//$parser->onParserStop = function($parser){};
//$parser->onParserMessage = function($parser, $connection, $download_data){};
//$parser->onParserFindUrl = function($parser, $sub_url){};
}
function startAppServer()
{
$server = new Server();
$server->onServerStart = function(){
//execute the task every 1 second
new Crontab('*/1 * * * * *', function(){
pprint("print the current time every 1 second: " . time());
});
//execute the task every 2 minutes
new Crontab('*/2 * * * *', function(){
pprint("print the current time every 2 minutes: " . time());
});
};
}
//start producer component
startAppProducer();
//start downloader component
startAppDownloader();
//start parser component
startAppParser();
//start server component
startAppServer();
//start phpcreeper engine
PHPCreeper::start();
$downloader->onAfterDownloader = function($downloader){
//dbo single instance and we can pass the DSN string `test`
$downloader->getDbo('test')->select('title', '*');
//dbo single instance and we can pass the configuration array
$config = Configurator::get('globalConfig/database/dbo/test')
$downloader->getDbo($config)->select('title', '*');
//dbo new instance and we can pass the DSN string `test`
$downloader->newDbo('test')->select('title', '*');
//dbo new instance and we can pass the configuration array
$config = Configurator::get('globalConfig/database/dbo/test')
$downloader->newDbo($config)->select('title', '*');
};
/path/to/php weather.php start
php start.php start
php start.php start -d
php start.php stop
php start.php restart
php start.php reload
php start.php reload -g
php start.php status
php start.php connections
Loading please wait ...
Before you can download the PHP files, the dependencies should be resolved. This can take some minutes. Please be patient.