Ejemplo n.º 1
0
 /**
  *
  */
 public function importData($url, $program_id, $feed_id, $custom_categorie)
 {
     $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
     $this->downloadAndSaveFeed($url, $fileLocation);
     $this->filterBestand($fileLocation);
     $chunkSize = Config::get('daisycon.chunksize', 500);
     Excel::filter('chunk')->load($fileLocation)->chunk($chunkSize, function ($results) use($program_id, $feed_id, $custom_categorie) {
         foreach ($results as $row) {
             /**
              * Lege values eruit filteren
              */
             $arr = array_filter($row->toArray());
             try {
                 /**
                  * Merge 'program_id' in gegevens uit XML
                  */
                 $inserted_array = array_merge($arr, array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
                 Data::create($inserted_array);
             } catch (Exception $e) {
                 dd($e->getMessage());
             }
         }
     });
     Data::where(function ($query) {
         $query->whereTitle('title')->orWhere('title', 'like', '#%');
     })->delete();
     Data::whereTemp(0)->update(array('temp' => 1));
     \File::delete($fileLocation);
 }
Ejemplo n.º 2
0
 /**
  * @param $url
  * @param $program_id
  * @param $feed_id
  * @param $custom_categorie
  *
  * @throws \Exception
  */
 public function importData($url, $program_id, $feed_id, $custom_categorie)
 {
     $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
     $this->downloadAndSaveFeed($url, $fileLocation);
     $this->filterBestand($fileLocation);
     $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
     $offset = 1;
     // initieel op 1 om header te ontlopen
     $batchAantal = 1000;
     $csv = Reader::createFromPath($fileLocation);
     $csv->setDelimiter(';');
     $csv->setEnclosure('"');
     $creationCount = 0;
     while (true) {
         // Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken
         \DB::connection()->flushQueryLog();
         $csv->setOffset($offset)->setLimit($batchAantal);
         $this->console->writeln("Memory now at: " . memory_get_peak_usage());
         $csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) {
             if (count($row) != count($fields_wanted_from_config)) {
                 return;
             }
             try {
                 $inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
                 Data::create($inserted_array);
                 $creationCount++;
             } catch (Exception $e) {
                 echo $e->getMessage() . PHP_EOL;
             } catch (\ErrorException $e) {
                 echo $e->getMessage() . PHP_EOL;
             }
         });
         $aantalResultaten = count($csvResults);
         $this->console->writeln("Totaal verwerkt: " . $creationCount);
         $offset += $aantalResultaten;
         if ($aantalResultaten != $batchAantal) {
             break;
         }
         // forceer einde
     }
     Data::where(function ($query) {
         $query->whereTitle('title')->orWhere('title', 'like', '#%');
     })->delete();
     Data::whereTemp(null)->update(array('temp' => 1));
     \File::delete($fileLocation);
 }
Ejemplo n.º 3
0
 /**
  * Importeer data van betreffende feed (url) in de database
  *
  * @param $url
  * @param $program_id
  * @param $feed_id
  * @param $custom_categorie
  */
 public function importData($url, $program_id, $feed_id, $custom_categorie)
 {
     $client = new Client();
     $response = $client->request('GET', $url, ['timeout' => 3]);
     $body = $response->getBody();
     $simpleXmlString = simplexml_load_string($body, null, LIBXML_NOCDATA);
     // LIBXML_NOCDATA-trick from: http://dissectionbydavid.wordpress.com/2013/01/25/simple-simplexml-to-array-in-php/
     foreach ($simpleXmlString as $simpleXmlNode) {
         // Lege values eruit filteren
         $arr = array_filter((array) $simpleXmlNode);
         try {
             // Merge 'program_id' in gegevens uit XML
             $inserted_array = array_merge($arr, array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
             Data::create($inserted_array);
         } catch (Exception $e) {
             dd($e->getMessage());
         }
     }
     return;
 }