/**
* Clean collection by filling in all the blanks.
*/
public function clean() : Collection
{
Log::notice(sprintf('Started validating %d entry(ies).', $this->entries->count()));
$newCollection = new Collection();
/** @var ImportEntry $entry */
foreach ($this->entries as $index => $entry) {
Log::debug(sprintf('--- import validator start for row %d ---', $index));
/*
* X Adds the date (today) if no date is present.
* X Determins the types of accounts involved (asset, expense, revenue).
* X Determins the type of transaction (withdrawal, deposit, transfer).
* - Determins the currency of the transaction.
* X Adds a default description if there isn't one present.
*/
$entry = $this->checkAmount($entry);
$entry = $this->setDate($entry);
$entry = $this->setAssetAccount($entry);
$entry = $this->setOpposingAccount($entry);
$entry = $this->cleanDescription($entry);
$entry = $this->setTransactionType($entry);
$entry = $this->setTransactionCurrency($entry);
$newCollection->put($index, $entry);
$this->job->addStepsDone(1);
}
Log::notice(sprintf('Finished validating %d entry(ies).', $newCollection->count()));
return $newCollection;
}
/**
* Run the actual import
*
* @return Collection
*/
public function createImportEntries() : Collection
{
$config = $this->job->configuration;
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$results = $reader->fetch();
Log::notice('Building importable objects from CSV file.');
foreach ($results as $index => $row) {
if ($index >= $start) {
$line = $index + 1;
Log::debug('----- import entry build start --');
Log::debug(sprintf('Now going to import row %d.', $index));
$importEntry = $this->importSingleRow($index, $row);
$this->collection->put($line, $importEntry);
/**
* 1. Build import entry.
* 2. Validate import entry.
* 3. Store journal.
* 4. Run rules.
*/
$this->job->addTotalSteps(4);
$this->job->addStepsDone(1);
}
}
Log::debug(sprintf('Import collection contains %d entries', $this->collection->count()));
Log::notice(sprintf('Built %d importable object(s) from your CSV file.', $this->collection->count()));
return $this->collection;
}
/**
* @param string $fileType
*
* @return ImportJob
*/
public function create(string $fileType) : ImportJob
{
$count = 0;
$fileType = strtolower($fileType);
$keys = array_keys(config('firefly.import_formats'));
if (!in_array($fileType, $keys)) {
throw new FireflyException(sprintf('Cannot use type "%s" for import job.', $fileType));
}
while ($count < 30) {
$key = Str::random(12);
$existing = $this->findByKey($key);
if (is_null($existing->id)) {
$importJob = new ImportJob();
$importJob->user()->associate($this->user);
$importJob->file_type = $fileType;
$importJob->key = Str::random(12);
$importJob->status = 'import_status_never_started';
$importJob->extended_status = ['total_steps' => 0, 'steps_done' => 0, 'import_count' => 0, 'importTag' => 0, 'errors' => []];
$importJob->save();
// breaks the loop:
return $importJob;
}
$count++;
}
return new ImportJob();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$jobKey = $this->argument('key');
$job = ImportJob::whereKey($jobKey)->first();
if (!$this->isValid($job)) {
return;
}
$this->line('Going to import job with key "' . $job->key . '" of type ' . $job->file_type);
$monolog = Log::getMonolog();
$handler = new CommandHandler($this);
$monolog->pushHandler($handler);
$result = ImportProcedure::runImport($job);
/**
* @var int $index
* @var TransactionJournal $journal
*/
foreach ($result as $index => $journal) {
if (!is_null($journal->id)) {
$this->line(sprintf('Line #%d has been imported as transaction #%d.', $index, $journal->id));
continue;
}
$this->error(sprintf('Could not store line #%d', $index));
}
$this->line('The import has completed.');
// get any errors from the importer:
$extendedStatus = $job->extended_status;
if (isset($extendedStatus['errors']) && count($extendedStatus['errors']) > 0) {
$this->line(sprintf('The following %d error(s) occured during the import:', count($extendedStatus['errors'])));
foreach ($extendedStatus['errors'] as $error) {
$this->error($error);
}
}
return;
}
/**
* @param ImportJob $job
*
* @return Collection
*/
public static function runImport(ImportJob $job) : Collection
{
// update job to say we started.
$job->status = 'import_running';
$job->save();
// create Importer
$valid = array_keys(config('firefly.import_formats'));
$class = 'INVALID';
if (in_array($job->file_type, $valid)) {
$class = config('firefly.import_formats.' . $job->file_type);
}
/** @var ImporterInterface $importer */
$importer = app($class);
$importer->setJob($job);
// create import entries
$collection = $importer->createImportEntries();
// validate / clean collection:
$validator = new ImportValidator($collection);
$validator->setUser($job->user);
$validator->setJob($job);
if ($job->configuration['import-account'] != 0) {
$repository = app(AccountCrud::class, [$job->user]);
$validator->setDefaultImportAccount($repository->find($job->configuration['import-account']));
}
$cleaned = $validator->clean();
// then import collection:
$storage = new ImportStorage($job->user, $cleaned);
$storage->setJob($job);
// and run store routine:
$result = $storage->store();
// grab import tag:
$status = $job->extended_status;
$status['importTag'] = $storage->importTag->id;
$job->extended_status = $status;
$job->status = 'import_complete';
$job->save();
return $result;
}
/**
* @return array
*/
private function getDataForColumnRoles() : array
{
$config = $this->job->configuration;
$data = ['columns' => [], 'columnCount' => 0];
// show user column role configuration.
$content = $this->job->uploadFileContents();
// create CSV reader.
$reader = Reader::createFromString($content);
$reader->setDelimiter($config['delimiter']);
$start = $config['has-headers'] ? 1 : 0;
$end = $start + config('csv.example_rows');
// collect example data in $data['columns']
while ($start < $end) {
$row = $reader->fetchOne($start);
// run specifics here:
// and this is the point where the specifix go to work.
foreach ($config['specifics'] as $name => $enabled) {
/** @var SpecificInterface $specific */
$specific = app('FireflyIII\\Import\\Specifics\\' . $name);
// it returns the row, possibly modified:
$row = $specific->run($row);
}
foreach ($row as $index => $value) {
$value = trim($value);
if (strlen($value) > 0) {
$data['columns'][$index][] = $value;
}
}
$start++;
$data['columnCount'] = count($row) > $data['columnCount'] ? count($row) : $data['columnCount'];
}
// make unique example data
foreach ($data['columns'] as $index => $values) {
$data['columns'][$index] = array_unique($values);
}
$data['set_roles'] = [];
// collect possible column roles:
$data['available_roles'] = [];
foreach (array_keys(config('csv.import_roles')) as $role) {
$data['available_roles'][$role] = trans('csv.column_' . $role);
}
$config['column-count'] = $data['columnCount'];
$this->job->configuration = $config;
$this->job->save();
return $data;
}
/**
* @param int $index
* @param ImportEntry $entry
*
* @return TransactionJournal
* @throws FireflyException
*/
private function storeSingle(int $index, ImportEntry $entry) : TransactionJournal
{
if ($entry->valid === false) {
Log::warning(sprintf('Cannot import row %d, because the entry is not valid.', $index));
$errors = join(', ', $entry->errors->all());
$errorText = sprintf('Row #%d: ' . $errors, $index);
$extendedStatus = $this->job->extended_status;
$extendedStatus['errors'][] = $errorText;
$this->job->extended_status = $extendedStatus;
$this->job->save();
return new TransactionJournal();
}
$alreadyImported = $this->alreadyImported($entry->hash);
if (!is_null($alreadyImported->id)) {
Log::warning(sprintf('Cannot import row %d, because it has already been imported (journal #%d).', $index, $alreadyImported->id));
$errorText = trans('firefly.import_double', ['row' => $index, 'link' => route('transactions.show', [$alreadyImported->id]), 'description' => $alreadyImported->description]);
$extendedStatus = $this->job->extended_status;
$extendedStatus['errors'][] = $errorText;
$this->job->extended_status = $extendedStatus;
$this->job->save();
return new TransactionJournal();
}
Log::debug(sprintf('Going to store row %d', $index));
$journal = $this->storeJournal($entry);
$amount = $this->makePositive($entry->fields['amount']);
$accounts = $this->storeAccounts($entry);
// create new transactions. This is something that needs a rewrite for multiple/split transactions.
$sourceData = ['account_id' => $accounts['source']->id, 'transaction_journal_id' => $journal->id, 'description' => $journal->description, 'amount' => bcmul($amount, '-1')];
$destinationData = ['account_id' => $accounts['destination']->id, 'transaction_journal_id' => $journal->id, 'description' => $journal->description, 'amount' => $amount];
$one = Transaction::create($sourceData);
$two = Transaction::create($destinationData);
$error = false;
if (is_null($one->id)) {
Log::error('Could not create transaction 1.', $one->getErrors()->all());
$error = true;
}
if (is_null($two->id)) {
Log::error('Could not create transaction 1.', $two->getErrors()->all());
$error = true;
}
// respond to error
if ($error === true) {
$errorText = sprintf('Cannot import row %d, because an error occured when storing data.', $index);
Log::error($errorText);
$extendedStatus = $this->job->extended_status;
$extendedStatus['errors'][] = $errorText;
$this->job->extended_status = $extendedStatus;
$this->job->save();
return new TransactionJournal();
}
Log::debug('Created transaction 1', ['id' => $one->id, 'account' => $one->account_id, 'account_name' => $accounts['source']->name]);
Log::debug('Created transaction 2', ['id' => $two->id, 'account' => $two->account_id, 'account_name' => $accounts['destination']->name]);
$journal->completed = 1;
$journal->save();
// attach import tag.
$journal->tags()->save($this->importTag);
// now attach budget and so on.
$this->storeBudget($journal, $entry);
$this->storeCategory($journal, $entry);
$this->storeBill($journal, $entry);
return $journal;
}
