PHP code example of rgilyov / laravel-csv-importer
1. Go to this page and download the library: Download rgilyov/laravel-csv-importer library. Choose the download type require. 2. Extract the ZIP file and open the index.php. 3. Add this code to the index.php.
<?php
require_once('vendor/autoload.php');
/* Start to develop here. Best regards https://php-download.com/ */
rgilyov / laravel-csv-importer example snippets
composer
'providers' => [
//...
\RGilyov\CsvImporter\CsvImporterServiceProvider::class,
];
namespace App\CsvImporters;
use RGilyov\CsvImporter\BaseCsvImporter;
class MyImporter extends BaseCsvImporter
{
/**
* Specify mappings and rules for the csv importer, you also may create csv files to write csv entities
* and overwrite global configurations
*
* @return array
*/
public function csvConfigurations()
{
return [
'mappings' => [
'serial_number' => [' * @throws \RGilyov\CsvImporter\Exceptions\CsvImporterException
* @return void
*/
public function handle($item)
{
$this->insertTo('valid_entities', $item);
}
/**
* Will be executed if a csv line did not pass validation
*
* @param $item
* @throws \RGilyov\CsvImporter\Exceptions\CsvImporterException
* @return void
*/
public function invalid($item)
{
$this->insertTo('invalid_entities', $item);
}
}
$importer = (new \App\CsvImporters\MyImporter())->setCsvFile('my_huge_csv_with_1000k_lines.csv');
$importer->run();
// progress information will be here, due to the import process already started above
$result = $importer->run();
$progress = $importer->getProgress();
/*
[
'data' => ["message" => 'The import process is running'],
'meta' => [
'processed' => 250000,
'remains' => 750000,
'percentage' => 25,
'finished' => false,
'init' => false,
'running' => true
]
]
*/
$finishDetails = $importer->finish();
/*
[
[
'data' => [
"message" => 'The import process successfully finished.'
],
'meta' => ["finished" => true, 'init' => false, 'running' => false],
'csv_files' => [
'valid_entities.csv',
'invalid_entities.csv'
]
]
]
*/
$importer->cancel();
namespace App\CsvImporters;
use RGilyov\CsvImporter\BaseCsvImporter;
class MyImporter extends BaseCsvImporter
{
//...
/**
* Will be executed before importing
*
* @return void
*/
protected function before()
{
// do something before the import start
if (! $this->checkSomething()) {
$this->setError('Oops', 'something went wrong.');
};
}
/**
* Adjust additional information to progress bar during import process
*
* @return null|string|array
*/
public function progressBarDetails()
{
return "I'm a csv importer and I'm running :)";
}
/**
* Will be executed after importing
*
* @return void
*/
protected function after()
{
// do something after the import finished
$entities = \App\CsvEntity::all(); // just a demo, in real life you don't want to do it ;)
$this->initProgressBar('Something running.', $entities->count());
$entities->each(function ($entity) {
// do something
$this->incrementProgress();
});
$this->setFinalDetails('Final details.');
}
/**
* Will be executed during the import process canceling
*/
protected function onCancel()
{
\DB::rollBack();
}
}
$importer = (new \App\CsvImporters\MyImporter())->setCsvFile('my_huge_csv_with_1000k_lines.csv');
$quantity = $importer->countCsv(); // returns amount of csv lines without headers
$distinctNames = $importer->distinct('name'); // returns array with distinct names
$importer->each(function ($item) { // encoded and casted csv line
// do something
});
/*
|--------------------------------------------------------------------------
| Main csv import configurations
|--------------------------------------------------------------------------
|
| `cache_driver` - keeps all progress and final information, it also allows
| the mutex functionality to work, there are only 3 cache drivers supported:
| redis, file and memcached
|
| `mutex_lock_time` - how long script will be executed and how long
| the import process will be locked, another words if we will import
| list of electric guitars we won't be able to run another import of electric
| guitars at the same time, to avoid duplicates and different sorts of
| incompatibilities. The value set in minutes.
|
| `memory_limit` - if you want store all csv values in memory or something like that,
| you may increase amount of memory for the script
|
| `encoding` - which encoding we have, UTF-8 by default
|
*/
'cache_driver' => env('CACHE_DRIVER', 'file'),
'mutex_lock_time' => 300,
'memory_limit' => 128,
/*
* An import class's short name (without namespace) by default
*/
'mutex_lock_key' => null,
/*
* Encoding of given csv file
*/
'input_encoding' => 'UTF-8',
/*
* Encoding of processed csv values
*/
'output_encoding' => 'UTF-8',
/*
* Specify which date format the given csv file has
* to use `date` ('Y-m-d') and `datetime` ('Y-m-d H:i:s') casters,
* if the parameter will be set to `null` `date` caster will replace
* `/` and `\` and `|` and `.` and `,` on `-` and will assume that
* the given csv file has `Y-m-d` or `d-m-Y` date format
*/
'csv_date_format' => null,
'delimiter' => ',',
'enclosure' => '"',
/*
* Warning: The library depends on PHP SplFileObject class.
* Since this class exhibits a reported bug (https://bugs.php.net/bug.php?id=55413),
* Data using the escape character are correctly
* escaped but the escape character is not removed from the CSV content.
*/
'escape' => '\\',
'newline' => "\n",
/*
|--------------------------------------------------------------------------
| Progress bar messages
|--------------------------------------------------------------------------
*/
'does_not_running' => 'Import process does not run',
'initialization' => 'Initialization',
'progress' => 'Import process is running',
'final_stage' => 'Final stage',
'finished' => 'Almost done, please click to the `finish` button to proceed',
'final' => 'The import process successfully finished!'
namespace App\CsvImporters;
use RGilyov\CsvImporter\BaseCsvImporter;
class MyImporter extends BaseCsvImporter
{
/**
* Specify mappings and rules for the csv importer, you also may create csv files to write csv entities
* and overwrite global configurations
*
* @return array
*/
public function csvConfigurations()
{
return [
'mappings' => [//...],
'csv_files' => [//...],
'config' => [
'mutex_lock_time' => 500,
'memory_limit' => 256,
'mutex_lock_key' => 'my-key',
'input_encoding' => 'cp1252',
'output_encoding' => 'UTF-8',
'csv_date_format' => 'm/d/Y',
'delimiter' => ';',
'enclosure' => '\'',
'escape' => '\\',
'newline' => "\n",
/*
|--------------------------------------------------------------------------
| Progress bar messages
|--------------------------------------------------------------------------
*/
'does_not_running' => 'Something does not run',
'initialization' => 'Init',
'progress' => 'Something running',
'final_stage' => 'After the import had finished',
'finished' => 'Please click to the `finish` button to proceed',
'final' => 'Something successfully finished!'
]
];
}
}
(new \App\CsvImporters\MyImporter())
->setCsvFile('my_huge_csv_with_1000k_lines.csv')
->setCsvDateFormat('y-m-d')
->setDelimiter('d')
->setEnclosure('e')
->setEscape("x")
->setInputEncoding('cp1252')
->setOutputEncoding('UTF-8')
->setNewline('newline')
->run();
class GuitarsCsvImporter extends BaseCsvImporter
{
/**
* Specify mappings and rules for the csv importer, you also may create csv files to write csv entities
* and overwrite global configurations
*
* @return array
*/
public function csvConfigurations()
{
return [
'mappings' => [
'name' => [] // <- defined mappings, we only need data from this column
]
];
}
/**
* Will be executed for a csv line if it passed validation
*
* @param $item
* @throws \RGilyov\CsvImporter\Exceptions\CsvImporterException
* @return void
*/
public function handle($item)
{
/*
$item contains ['name' => 'John', 'some_weird_header' => 'some_weird_data']
so the $item will have all columns inside, so we need extract only columns we need, which was defined
inside csv configurations mappings array
*/
$dataOnlyFromDefinedFields = $this->extractDefinedFields($item); // will return ['name' => 'John']
/*
Assume we need to do some manipulations with the $item
array and then write it to the `valid_entities.csv`
we need to make sure that data inside formatted array are
match headers inside the csv, we can do it with this `toCsvHeaders($item)` method:
*/
// will return ['name' => 'John', 'some_weird_header' => null]
$csvHeadersData = $this->toCsvHeaders($dataOnlyFromDefinedFields);
$this->insertTo('valid_entities', $csvHeadersData);
}
}
class GuitarsCsvImporter extends BaseCsvImporter
{
//..
protected $guitarCompany;
public function setCompany(\App\GuitarCompany $guitarCompany)
{
$this->guitarCompany = $guitarCompany;
$this->concatMutexKey($guitarCompany->id);
return $this;
}
/**
* Will be executed for a csv line if it passed validation
*
* @param $item
* @throws \RGilyov\CsvImporter\Exceptions\CsvImporterException
* @return void
*/
public function handle($item)
{
\App\Guitars::create(
array_merge(['guitar_company_id' => $this->guitarCompany->id], $this->extractDefinedFields($item))
);
}
}
namespace App\CsvImporters\HeadersFilters;
use RGilyov\CsvImporter\BaseHeadersFilter;
class MyHeadersFilter extends BaseHeadersFilter
{
/**
* Specify error message
*
* @var string
*/
public $errorMessage = 'The csv must contain either `name` field either `first_name` and `last_name` fields';
/**
* @param array $csvHeaders
* @return bool
*/
public function filter(array $csvHeaders)
{
if (isset($csvHeaders['name']) || (isset($csvHeaders['first_name']) && isset($csvHeaders['last_name']))) {
return true;
}
return false;
}
}
\App\CsvImporters\MyImporter::addHeadersFilter(new \App\CsvImporters\HeadersFilters\MyHeadersFilter());
// you may also use closure, you can specify name by passing second argument, otherwise it will be called `filter`
\App\CsvImporters\MyImporter::addHeadersFilter(function ($csvHeaders) {
if (isset($csvHeaders['A']) || isset($csvHeaders['B'])) {
return true;
}
return false;
}, 'ab-filter');
// you may add multiple filters with one method
$ABFilter = function ($csvHeaders) {
if (isset($csvHeaders['A']) || isset($csvHeaders['B'])) {
return true;
}
return false;
}
$myHeadersFilter = new \App\CsvImporters\HeadersFilters\MyHeadersFilter();
\App\CsvImporters\MyImporter::addHeadersFilters($ABFilter, $myHeadersFilter);
\App\CsvImporters\MyImporter::headersFilterExists('MyHeadersFilter'); // will return `true`
\App\CsvImporters\MyImporter::getHeadersFilter('MyHeadersFilter'); // will return the filter object
\App\CsvImporters\MyImporter::getHeadersFilters(); // will return array with all filter objects
\App\CsvImporters\MyImporter::unsetHeadersFilter('MyHeadersFilter'); // will return `true`
\App\CsvImporters\MyImporter::flushHeadersFilters(); // will return empty array
// example case
if ($request->get('without_filters')) {
\App\CsvImporters\MyImporter::flushHeadersFilters();
}
namespace App\CsvImporters\ValidationFilters;
use RGilyov\CsvImporter\BaseValidationFilter;
class MyValidationFilter extends BaseValidationFilter
{
/**
* @var string
*/
protected $name = 'bad_word_validation';
/**
* @param mixed $value
* @return bool
*/
public function filter($value)
{
if (strpos($value, 'bad_word') !== false) {
return false;
}
return true;
}
}
namespace App\CsvImporters\ValidationFilters;
use RGilyov\CsvImporter\BaseValidationFilter;
class MyGlobalValidationFilter extends BaseValidationFilter
{
/**
* @var string
*/
protected $name = 'global_validation';
/**
* @var string
*/
public $global = true;
/**
* @param mixed $value
* @return bool
*/
public function filter($value) // we will get array here, coz $this->global set to `true`
{
if (empty($value['name']) || (empty($value['first_name']) && empty($value['last_name']))) {
return false;
}
return true;
}
}
\App\CsvImporters\MyImporter::addValidationFilter(new \App\CsvImporters\ValidationFilters\MyValidationFilter());
// closure validation filters are global
\App\CsvImporters\MyImporter::addValidationFilter(function ($item) {
if (!empty($csvHeaders['A']) || !empty($csvHeaders['B'])) {
return true;
}
return false;
}, 'not-empty');
// you may add multiple filters with one method
$notEmpty = function ($item) {
if (!empty($item['A']) || !empty($item['B'])) {
return true;
}
return false;
}
$myValidationFilter = new \App\CsvImporters\ValidationFilters\MyValidationFilter();
$myGlobalValidationFilter = new \App\CsvImporters\ValidationFilters\MyGlobalValidationFilter();
\App\CsvImporters\MyImporter::addValidationFilters($notEmpty, $myValidationFilter, $myGlobalValidationFilter);
/////////////////////////////////////////////////////////////////////////////
\App\CsvImporters\MyImporter::validationFilterExists('bad_word_validation'); // will return `true`
\App\CsvImporters\MyImporter::getValidationFilter('global_validation'); // will return instance of MyGlobalValidationFilter class
\App\CsvImporters\MyImporter::getValidationFilters(); // will return array with all filter objects
\App\CsvImporters\MyImporter::unsetValidationFilter('bad_word_validation'); // will return `true`
\App\CsvImporters\MyImporter::flushValidationFilters(); // will return empty array
// example case
if ($request->get('without_filters')) {
\App\CsvImporters\MyImporter::flushValidationFilters();
}
namespace App\CsvImporters\CastFilters;
use RGilyov\CsvImporter\BaseCastFilter;
class MyCastFilter extends BaseCastFilter
{
protected $name = 'lowercase';
/**
* @param $value
* @return mixed
*/
public function filter($value)
{
return strtolower($value);
}
}
\App\CsvImporters\MyImporter::addCastFilter(new \App\CsvImporters\CastFilters\MyCastFilter());
\App\CsvImporters\MyImporter::addCastFilter(function ($value) {
return htmlspecialchars($value);
});
$htmlentities = function ($value) {
return htmlentities($value);
}
$myCastFilter = new \App\CsvImporters\CastFilters\MyCastFilter();
\App\CsvImporters\MyImporter::addCastFilters($htmlentities, $myCastFilter);
/////////////////////////////////////////////////////////////////////////////
\App\CsvImporters\MyImporter::castFilterExists('lowercase'); // will return `true`
\App\CsvImporters\MyImporter::getCastFilter('lowercase'); // will return instance of MyCastFilter class
\App\CsvImporters\MyImporter::getCastFilters(); // will return array with all filter objects
\App\CsvImporters\MyImporter::unsetCastFilter('lowercase'); // will return `true`
\App\CsvImporters\MyImporter::flushCastFilters(); // will return empty array
// example case
if ($request->get('without_filters')) {
\App\CsvImporters\MyImporter::flushCastFilters();
}
php artisan vendor:publish --tag=config
WARNING!!!
All closure validation filters are global.