/** * Execute the console command. * * @return mixed */ public function fire() { $this->info('Truncate data table'); Data::truncate(); $this->info('Searching for feeds'); $activeProgramsFromDB = ActiveProgram::with('program.feeds')->where('status', 1)->get(); $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport(); if (count($activeProgramsFromDB) > 0) { foreach ($activeProgramsFromDB as $activeProgram) { if (!empty($activeProgram->program->feeds) || !empty($activeProgram->program->name)) { foreach ($activeProgram->program->feeds as $feed) { $this->info($activeProgram->program->name . ' - ' . $feed->name); $url = $feed->{"feed_link_" . strtolower(Config::get('daisycon.feed_type', 'csv'))} . '&f=' . implode(';', $fields_wanted_from_config) . '&encoding=' . Config::get("daisycon.encoding") . '&general=true' . '&nohtml=' . (Config::get("daisycon.accept_html", false) ? 'false' : 'true'); $program_id = $activeProgram->program->program_id; $feed_id = $feed->feed_id; $custom_categorie = $activeProgram->custom_categorie; $this->data->importData($url, $program_id, $feed_id, $custom_categorie); } } else { $this->info('Geen feeds en/of programma\'s in de database gevonden...'); continue; } } } else { return $this->info('Geen active programma\'s in de database gevonden...'); } $this->call('daisycon:fix-data'); return $this->info('Verwerkt in ' . round(microtime(true) - LARAVEL_START, 2) . ' seconden'); }
/** * @param $url * @param $program_id * @param $feed_id * @param $custom_categorie * * @throws \Exception */ public function importData($url, $program_id, $feed_id, $custom_categorie) { $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv'; $this->downloadAndSaveFeed($url, $fileLocation); $this->filterBestand($fileLocation); $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport(); $offset = 1; // initieel op 1 om header te ontlopen $batchAantal = 1000; $csv = Reader::createFromPath($fileLocation); $csv->setDelimiter(';'); $csv->setEnclosure('"'); $creationCount = 0; while (true) { // Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken \DB::connection()->flushQueryLog(); $csv->setOffset($offset)->setLimit($batchAantal); $this->console->writeln("Memory now at: " . memory_get_peak_usage()); $csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) { if (count($row) != count($fields_wanted_from_config)) { return; } try { $inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie)); Data::create($inserted_array); $creationCount++; } catch (Exception $e) { echo $e->getMessage() . PHP_EOL; } catch (\ErrorException $e) { echo $e->getMessage() . PHP_EOL; } }); $aantalResultaten = count($csvResults); $this->console->writeln("Totaal verwerkt: " . $creationCount); $offset += $aantalResultaten; if ($aantalResultaten != $batchAantal) { break; } // forceer einde } Data::where(function ($query) { $query->whereTitle('title')->orWhere('title', 'like', '#%'); })->delete(); Data::whereTemp(null)->update(array('temp' => 1)); \File::delete($fileLocation); }