/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$this->info('Truncate data table');
Data::truncate();
$this->info('Searching for feeds');
$activeProgramsFromDB = ActiveProgram::with('program.feeds')->where('status', 1)->get();
$fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
if (count($activeProgramsFromDB) > 0) {
foreach ($activeProgramsFromDB as $activeProgram) {
if (!empty($activeProgram->program->feeds) || !empty($activeProgram->program->name)) {
foreach ($activeProgram->program->feeds as $feed) {
$this->info($activeProgram->program->name . ' - ' . $feed->name);
$url = $feed->{"feed_link_" . strtolower(Config::get('daisycon.feed_type', 'csv'))} . '&f=' . implode(';', $fields_wanted_from_config) . '&encoding=' . Config::get("daisycon.encoding") . '&general=true' . '&nohtml=' . (Config::get("daisycon.accept_html", false) ? 'false' : 'true');
$program_id = $activeProgram->program->program_id;
$feed_id = $feed->feed_id;
$custom_categorie = $activeProgram->custom_categorie;
$this->data->importData($url, $program_id, $feed_id, $custom_categorie);
}
} else {
$this->info('Geen feeds en/of programma\'s in de database gevonden...');
continue;
}
}
} else {
return $this->info('Geen active programma\'s in de database gevonden...');
}
$this->call('daisycon:fix-data');
return $this->info('Verwerkt in ' . round(microtime(true) - LARAVEL_START, 2) . ' seconden');
}
/**
* @param $url
* @param $program_id
* @param $feed_id
* @param $custom_categorie
*
* @throws \Exception
*/
public function importData($url, $program_id, $feed_id, $custom_categorie)
{
$fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
$this->downloadAndSaveFeed($url, $fileLocation);
$this->filterBestand($fileLocation);
$fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
$offset = 1;
// initieel op 1 om header te ontlopen
$batchAantal = 1000;
$csv = Reader::createFromPath($fileLocation);
$csv->setDelimiter(';');
$csv->setEnclosure('"');
$creationCount = 0;
while (true) {
// Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken
\DB::connection()->flushQueryLog();
$csv->setOffset($offset)->setLimit($batchAantal);
$this->console->writeln("Memory now at: " . memory_get_peak_usage());
$csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) {
if (count($row) != count($fields_wanted_from_config)) {
return;
}
try {
$inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
Data::create($inserted_array);
$creationCount++;
} catch (Exception $e) {
echo $e->getMessage() . PHP_EOL;
} catch (\ErrorException $e) {
echo $e->getMessage() . PHP_EOL;
}
});
$aantalResultaten = count($csvResults);
$this->console->writeln("Totaal verwerkt: " . $creationCount);
$offset += $aantalResultaten;
if ($aantalResultaten != $batchAantal) {
break;
}
// forceer einde
}
Data::where(function ($query) {
$query->whereTitle('title')->orWhere('title', 'like', '#%');
})->delete();
Data::whereTemp(null)->update(array('temp' => 1));
\File::delete($fileLocation);
}
