/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$this->info('Truncate data table');
Data::truncate();
$this->info('Searching for feeds');
$activeProgramsFromDB = ActiveProgram::with('program.feeds')->where('status', 1)->get();
$fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
if (count($activeProgramsFromDB) > 0) {
foreach ($activeProgramsFromDB as $activeProgram) {
if (!empty($activeProgram->program->feeds) || !empty($activeProgram->program->name)) {
foreach ($activeProgram->program->feeds as $feed) {
$this->info($activeProgram->program->name . ' - ' . $feed->name);
$url = $feed->{"feed_link_" . strtolower(Config::get('daisycon.feed_type', 'csv'))} . '&f=' . implode(';', $fields_wanted_from_config) . '&encoding=' . Config::get("daisycon.encoding") . '&general=true' . '&nohtml=' . (Config::get("daisycon.accept_html", false) ? 'false' : 'true');
$program_id = $activeProgram->program->program_id;
$feed_id = $feed->feed_id;
$custom_categorie = $activeProgram->custom_categorie;
$this->data->importData($url, $program_id, $feed_id, $custom_categorie);
}
} else {
$this->info('Geen feeds en/of programma\'s in de database gevonden...');
continue;
}
}
} else {
return $this->info('Geen active programma\'s in de database gevonden...');
}
$this->call('daisycon:fix-data');
return $this->info('Verwerkt in ' . round(microtime(true) - LARAVEL_START, 2) . ' seconden');
}
/**
* @param $url
* @param $program_id
* @param $feed_id
* @param $custom_categorie
*
* @throws \Exception
*/
public function importData($url, $program_id, $feed_id, $custom_categorie)
{
$fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv';
$this->downloadAndSaveFeed($url, $fileLocation);
$this->filterBestand($fileLocation);
$fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport();
$offset = 1;
// initieel op 1 om header te ontlopen
$batchAantal = 1000;
$csv = Reader::createFromPath($fileLocation);
$csv->setDelimiter(';');
$csv->setEnclosure('"');
$creationCount = 0;
while (true) {
// Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken
\DB::connection()->flushQueryLog();
$csv->setOffset($offset)->setLimit($batchAantal);
$this->console->writeln("Memory now at: " . memory_get_peak_usage());
$csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) {
if (count($row) != count($fields_wanted_from_config)) {
return;
}
try {
$inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie));
Data::create($inserted_array);
$creationCount++;
} catch (Exception $e) {
echo $e->getMessage() . PHP_EOL;
} catch (\ErrorException $e) {
echo $e->getMessage() . PHP_EOL;
}
});
$aantalResultaten = count($csvResults);
$this->console->writeln("Totaal verwerkt: " . $creationCount);
$offset += $aantalResultaten;
if ($aantalResultaten != $batchAantal) {
break;
}
// forceer einde
}
Data::where(function ($query) {
$query->whereTitle('title')->orWhere('title', 'like', '#%');
})->delete();
Data::whereTemp(null)->update(array('temp' => 1));
\File::delete($fileLocation);
}
/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$this->info('Start importing subscriptions');
// if ($path = $this->option($option))
// {
// return $path;
// }
// return Config::get("generators.{$configName}");
// return Config::get("generators.{$configName}");
// echo 'Config media_id: '. Config::get("daisycon");
// $media_id = Config::get("daisycon.media_id");
$sWsdl_program = "http://api.daisycon.com/publisher/soap/program/wsdl/";
$oSoapClient_program = new \SoapClient($sWsdl_program, DaisyconHelper::getApiOptions());
$aFilter = array();
try {
$mResult = $oSoapClient_program->getSubscriptions($aFilter);
} catch (Exception $e) {
// var_dump( $oSoapClient_program->__getLastRequestHeaders() );
// var_dump( $oSoapClient_program->__getLastResponse() );
return $this->error('Fout met binnenhalen van de \'subscriptions\'');
}
// print_r($mResult);
// return;
if (count($mResult['return']) > 0) {
// Truncate table
$this->info('Verwijderen van bestaande subscriptions');
Subscription::truncate();
foreach ($mResult['return'] as $subscription) {
$subscriptionArray = (array) $subscription;
$subscriptionArray['media'] = json_encode($subscriptionArray['media']);
// dd($subscriptionArray);
Subscription::create((array) $subscriptionArray);
}
// foreach mResult
} else {
$this->info('Geen subscriptions gevonden');
}
return $this->info('done');
}
/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$page = 1;
$per_page = 50;
$notLastPage = true;
$options = array('page' => $page, 'per_page' => $per_page, 'productfeed' => 'true');
$this->info('Start importing programs into the database');
while ($notLastPage) {
$APIdata = DaisyconHelper::getRestAPI("programs", $options);
if (is_array($APIdata)) {
$resultCount = count($APIdata['response']);
if ($resultCount > 0) {
if ($page == 1) {
$this->info('Clear database table');
Program::truncate();
}
foreach ($APIdata['response'] as $program) {
$program = (array) $program;
$program['program_id'] = $program['id'];
$program['description'] = $program['descriptions'][0]->description;
$program['url'] = DaisyconHelper::changeProgramURL($program['url']);
Program::create((array) $program);
}
$totalCount = Program::all()->count();
$comment = sprintf('Page %d loaded with %d record(s); Total records: %d', $page, $resultCount, $totalCount);
$this->comment($comment);
} else {
$this->comment('No programs found');
}
}
if ($resultCount < $per_page) {
$notLastPage = false;
}
$options['page'] = $page++;
}
$count = Program::all()->count();
return $this->info($count . ' programs imported');
}
/**
* Execute the console command.
*
* @return mixed
*/
public function fire()
{
$media_id = Config::get("daisycon.media_id");
$sub_id = Config::get("daisycon.sub_id");
$page = 1;
$per_page = 50;
$notLastPage = true;
$options = array('page' => $page, 'per_page' => $per_page, 'media_id' => $media_id, 'placeholder_media_id' => $media_id, 'placeholder_subid' => $sub_id);
$this->info('Start importing feeds into the database');
while ($notLastPage) {
$APIdata = DaisyconHelper::getRestAPI("productfeeds", $options);
if (is_array($APIdata)) {
$resultCount = count($APIdata['response']);
if ($resultCount > 0) {
if ($page == 1) {
$this->info('Truncate database table');
Feed::truncate();
}
foreach ($APIdata['response'] as $feedinfo) {
$feedinfo = (array) $feedinfo;
$feedinfo['feed_id'] = $feedinfo['id'];
Feed::create($feedinfo);
}
$totalCount = Feed::all()->count();
$comment = sprintf('Page %d loaded with %d record(s); Total records: %d', $page, $resultCount, $totalCount);
$this->comment($comment);
} else {
return $this->comment('Geen feeds gevonden');
}
}
if (isset($resultCount) && $resultCount < $per_page) {
$notLastPage = false;
}
$options['page'] = $page++;
}
$count = Feed::all()->count();
return $this->info($count . ' feeds imported');
}
public static function dataVoorbereiden($data)
{
$data = parent::fixTransportationType($data);
$data = parent::fixBoardingType($data);
$data = parent::fixLandcodes($data);
$data = parent::fixAccommodationName($data);
$data = parent::fixStars($data);
$data = parent::fixDescription($data);
$data = parent::fixPositions($data);
$data = parent::fixDuration($data);
// Slug aan het einde laten staan
foreach (DaisyconHelper::getDatabaseFields() as $fieldname) {
if (str_contains($fieldname, 'slug_')) {
$originalFieldName = str_replace('slug_', '', $fieldname);
if (!empty($data->{$originalFieldName})) {
$data->{$fieldname} = Str::slug($data->{$originalFieldName});
}
}
// Encoding aanpassen
$data->{$fieldname} = html_entity_decode($data->{$fieldname}, ENT_QUOTES, "utf-8");
// nog testen
// Trim alle velden
$data->{$fieldname} = trim($data->{$fieldname});
// Specialchars aanpassen
$regex = '/http:\\/\\/.*/';
if (isset($data->{$fieldname}) && preg_match($regex, $data->{$fieldname})) {
$data->{$fieldname} = htmlspecialchars_decode(urldecode($data->{$fieldname}));
// sneller dan html_entity_decode @ http://stackoverflow.com/questions/11723641/how-to-decode-the-amp-from-a-url-so-that-header-works-urldecode-not-working
// Brakke utm_ arguments vervangen in url's
$regex = '/(&utm_\\w+=%\\w+%)&/';
if (preg_match_all($regex, $data->{$fieldname}, $matches)) {
$data->{$fieldname} = str_replace($matches[1], '', $data->{$fieldname});
}
}
}
return $data;
}
