/**
*
*/
public function importData($url, $program_id, $feed_id, $custom_categorie)
{
$fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
$this->downloadAndSaveFeed($url, $fileLocation);
$this->filterBestand($fileLocation);
$chunkSize = Config::get('daisycon.chunksize', 500);
Excel::filter('chunk')->load($fileLocation)->chunk($chunkSize, function ($results) use($program_id, $feed_id, $custom_categorie) {
foreach ($results as $row) {
/**
* Lege values eruit filteren
*/
$arr = array_filter($row->toArray());
try {
/**
* Merge 'program_id' in gegevens uit XML
*/
$inserted_array = array_merge($arr, array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
Data::create($inserted_array);
} catch (Exception $e) {
dd($e->getMessage());
}
}
});
Data::where(function ($query) {
$query->whereTitle('title')->orWhere('title', 'like', '#%');
})->delete();
Data::whereTemp(0)->update(array('temp' => 1));
\File::delete($fileLocation);
}
/**
* @param $url
* @param $program_id
* @param $feed_id
* @param $custom_categorie
*
* @throws \Exception
*/
public function importData($url, $program_id, $feed_id, $custom_categorie)
{
$fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
$this->downloadAndSaveFeed($url, $fileLocation);
$this->filterBestand($fileLocation);
$fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
$offset = 1;
// initieel op 1 om header te ontlopen
$batchAantal = 1000;
$csv = Reader::createFromPath($fileLocation);
$csv->setDelimiter(';');
$csv->setEnclosure('"');
$creationCount = 0;
while (true) {
// Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken
\DB::connection()->flushQueryLog();
$csv->setOffset($offset)->setLimit($batchAantal);
$this->console->writeln("Memory now at: " . memory_get_peak_usage());
$csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) {
if (count($row) != count($fields_wanted_from_config)) {
return;
}
try {
$inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
Data::create($inserted_array);
$creationCount++;
} catch (Exception $e) {
echo $e->getMessage() . PHP_EOL;
} catch (\ErrorException $e) {
echo $e->getMessage() . PHP_EOL;
}
});
$aantalResultaten = count($csvResults);
$this->console->writeln("Totaal verwerkt: " . $creationCount);
$offset += $aantalResultaten;
if ($aantalResultaten != $batchAantal) {
break;
}
// forceer einde
}
Data::where(function ($query) {
$query->whereTitle('title')->orWhere('title', 'like', '#%');
})->delete();
Data::whereTemp(null)->update(array('temp' => 1));
\File::delete($fileLocation);
}
public function importData($url, $program_id, $feed_id, $custom_categorie)
{
$fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
$response = $this->downloadAndSaveFeed($url, $fileLocation);
if ($response) {
// $this->filterBestand($fileLocation);
$pdo = DB::connection()->getPdo();
$sql = "LOAD DATA INFILE '" . addslashes($fileLocation) . "'\r\n INTO TABLE `data`\r\n CHARACTER SET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'\r\n (`title`, `link`, `description`, `accommodation_name`, `accommodation_type`, `min_nr_people`, `location_description`, `stars`, `minimum_price`, `maximum_price`, `lowest_price`, `continent_of_destination`, `country_of_destination`, `country_link`, `region_of_destination`, `region_link`, `city_of_destination`, `city_link`, `longitude`, `latitude`, `continent_of_origin`, `country_of_origin`, `city_of_origin`, `port_of_departure`, `img_small`, `img_medium`, `img_large`, `board_type`, `tour_operator`, `transportation_type`, `departure-date`, `departure_date`, `end_date`, `duration`, `daisycon_unique_id`, `internal_id`, `unique_integer`, `update_hash`)\r\n SET\r\n `created_at` = NOW(),\r\n `updated_at` = NOW(),\r\n `program_id` = " . $program_id . ",\r\n `feed_id` = " . $feed_id . ",\r\n `custom_categorie` = '" . $custom_categorie . "'\r\n ";
// DB::connection()->getPdo()->exec($sql);
$pdo->exec($sql);
Data::where(function ($query) {
$query->whereTitle('title')->orWhere('title', 'like', '#%');
})->delete();
// DB::table('data')->update(array('temp' => 1));
Data::whereTemp(0)->update(array('temp' => 1));
\File::delete($fileLocation);
}
}
