/** * */ public function importData($url, $program_id, $feed_id, $custom_categorie) { $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv'; $this->downloadAndSaveFeed($url, $fileLocation); $this->filterBestand($fileLocation); $chunkSize = Config::get('daisycon.chunksize', 500); Excel::filter('chunk')->load($fileLocation)->chunk($chunkSize, function ($results) use($program_id, $feed_id, $custom_categorie) { foreach ($results as $row) { /** * Lege values eruit filteren */ $arr = array_filter($row->toArray()); try { /** * Merge 'program_id' in gegevens uit XML */ $inserted_array = array_merge($arr, array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie)); Data::create($inserted_array); } catch (Exception $e) { dd($e->getMessage()); } } }); Data::where(function ($query) { $query->whereTitle('title')->orWhere('title', 'like', '#%'); })->delete(); Data::whereTemp(0)->update(array('temp' => 1)); \File::delete($fileLocation); }
/** * @param $url * @param $program_id * @param $feed_id * @param $custom_categorie * * @throws \Exception */ public function importData($url, $program_id, $feed_id, $custom_categorie) { $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv'; $this->downloadAndSaveFeed($url, $fileLocation); $this->filterBestand($fileLocation); $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport(); $offset = 1; // initieel op 1 om header te ontlopen $batchAantal = 1000; $csv = Reader::createFromPath($fileLocation); $csv->setDelimiter(';'); $csv->setEnclosure('"'); $creationCount = 0; while (true) { // Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken \DB::connection()->flushQueryLog(); $csv->setOffset($offset)->setLimit($batchAantal); $this->console->writeln("Memory now at: " . memory_get_peak_usage()); $csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) { if (count($row) != count($fields_wanted_from_config)) { return; } try { $inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie)); Data::create($inserted_array); $creationCount++; } catch (Exception $e) { echo $e->getMessage() . PHP_EOL; } catch (\ErrorException $e) { echo $e->getMessage() . PHP_EOL; } }); $aantalResultaten = count($csvResults); $this->console->writeln("Totaal verwerkt: " . $creationCount); $offset += $aantalResultaten; if ($aantalResultaten != $batchAantal) { break; } // forceer einde } Data::where(function ($query) { $query->whereTitle('title')->orWhere('title', 'like', '#%'); })->delete(); Data::whereTemp(null)->update(array('temp' => 1)); \File::delete($fileLocation); }
/** * Importeer data van betreffende feed (url) in de database * * @param $url * @param $program_id * @param $feed_id * @param $custom_categorie */ public function importData($url, $program_id, $feed_id, $custom_categorie) { $client = new Client(); $response = $client->request('GET', $url, ['timeout' => 3]); $body = $response->getBody(); $simpleXmlString = simplexml_load_string($body, null, LIBXML_NOCDATA); // LIBXML_NOCDATA-trick from: http://dissectionbydavid.wordpress.com/2013/01/25/simple-simplexml-to-array-in-php/ foreach ($simpleXmlString as $simpleXmlNode) { // Lege values eruit filteren $arr = array_filter((array) $simpleXmlNode); try { // Merge 'program_id' in gegevens uit XML $inserted_array = array_merge($arr, array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie)); Data::create($inserted_array); } catch (Exception $e) { dd($e->getMessage()); } } return; }