/** * Execute the console command. * * @return mixed */ public function fire() { $this->info('Truncate data table'); Data::truncate(); $this->info('Searching for feeds'); $activeProgramsFromDB = ActiveProgram::with('program.feeds')->where('status', 1)->get(); $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport(); if (count($activeProgramsFromDB) > 0) { foreach ($activeProgramsFromDB as $activeProgram) { if (!empty($activeProgram->program->feeds) || !empty($activeProgram->program->name)) { foreach ($activeProgram->program->feeds as $feed) { $this->info($activeProgram->program->name . ' - ' . $feed->name); $url = $feed->{"feed_link_" . strtolower(Config::get('daisycon.feed_type', 'csv'))} . '&f=' . implode(';', $fields_wanted_from_config) . '&encoding=' . Config::get("daisycon.encoding") . '&general=true' . '&nohtml=' . (Config::get("daisycon.accept_html", false) ? 'false' : 'true'); $program_id = $activeProgram->program->program_id; $feed_id = $feed->feed_id; $custom_categorie = $activeProgram->custom_categorie; $this->data->importData($url, $program_id, $feed_id, $custom_categorie); } } else { $this->info('Geen feeds en/of programma\'s in de database gevonden...'); continue; } } } else { return $this->info('Geen active programma\'s in de database gevonden...'); } $this->call('daisycon:fix-data'); return $this->info('Verwerkt in ' . round(microtime(true) - LARAVEL_START, 2) . ' seconden'); }
/** * @param $url * @param $program_id * @param $feed_id * @param $custom_categorie * * @throws \Exception */ public function importData($url, $program_id, $feed_id, $custom_categorie) { $fileLocation = storage_path() . '/' . $program_id . '.' . $feed_id . '.csv'; $this->downloadAndSaveFeed($url, $fileLocation); $this->filterBestand($fileLocation); $fields_wanted_from_config = DaisyconHelper::getDatabaseFieldsToImport(); $offset = 1; // initieel op 1 om header te ontlopen $batchAantal = 1000; $csv = Reader::createFromPath($fileLocation); $csv->setDelimiter(';'); $csv->setEnclosure('"'); $creationCount = 0; while (true) { // Flushing the QueryLog anders kan de import te veel geheugen gaan gebruiken \DB::connection()->flushQueryLog(); $csv->setOffset($offset)->setLimit($batchAantal); $this->console->writeln("Memory now at: " . memory_get_peak_usage()); $csvResults = $csv->fetchAll(function ($row) use($fields_wanted_from_config, $program_id, $feed_id, $custom_categorie, &$creationCount) { if (count($row) != count($fields_wanted_from_config)) { return; } try { $inserted_array = array_merge(array_combine($fields_wanted_from_config, $row), array('program_id' => $program_id, 'feed_id' => $feed_id, 'custom_categorie' => $custom_categorie)); Data::create($inserted_array); $creationCount++; } catch (Exception $e) { echo $e->getMessage() . PHP_EOL; } catch (\ErrorException $e) { echo $e->getMessage() . PHP_EOL; } }); $aantalResultaten = count($csvResults); $this->console->writeln("Totaal verwerkt: " . $creationCount); $offset += $aantalResultaten; if ($aantalResultaten != $batchAantal) { break; } // forceer einde } Data::where(function ($query) { $query->whereTitle('title')->orWhere('title', 'like', '#%'); })->delete(); Data::whereTemp(null)->update(array('temp' => 1)); \File::delete($fileLocation); }
/** * Execute the console command. * * @return mixed */ public function fire() { $this->info('Start importing subscriptions'); // if ($path = $this->option($option)) // { // return $path; // } // return Config::get("generators.{$configName}"); // return Config::get("generators.{$configName}"); // echo 'Config media_id: '. Config::get("daisycon"); // $media_id = Config::get("daisycon.media_id"); $sWsdl_program = "http://api.daisycon.com/publisher/soap/program/wsdl/"; $oSoapClient_program = new \SoapClient($sWsdl_program, DaisyconHelper::getApiOptions()); $aFilter = array(); try { $mResult = $oSoapClient_program->getSubscriptions($aFilter); } catch (Exception $e) { // var_dump( $oSoapClient_program->__getLastRequestHeaders() ); // var_dump( $oSoapClient_program->__getLastResponse() ); return $this->error('Fout met binnenhalen van de \'subscriptions\''); } // print_r($mResult); // return; if (count($mResult['return']) > 0) { // Truncate table $this->info('Verwijderen van bestaande subscriptions'); Subscription::truncate(); foreach ($mResult['return'] as $subscription) { $subscriptionArray = (array) $subscription; $subscriptionArray['media'] = json_encode($subscriptionArray['media']); // dd($subscriptionArray); Subscription::create((array) $subscriptionArray); } // foreach mResult } else { $this->info('Geen subscriptions gevonden'); } return $this->info('done'); }
/** * Execute the console command. * * @return mixed */ public function fire() { $page = 1; $per_page = 50; $notLastPage = true; $options = array('page' => $page, 'per_page' => $per_page, 'productfeed' => 'true'); $this->info('Start importing programs into the database'); while ($notLastPage) { $APIdata = DaisyconHelper::getRestAPI("programs", $options); if (is_array($APIdata)) { $resultCount = count($APIdata['response']); if ($resultCount > 0) { if ($page == 1) { $this->info('Clear database table'); Program::truncate(); } foreach ($APIdata['response'] as $program) { $program = (array) $program; $program['program_id'] = $program['id']; $program['description'] = $program['descriptions'][0]->description; $program['url'] = DaisyconHelper::changeProgramURL($program['url']); Program::create((array) $program); } $totalCount = Program::all()->count(); $comment = sprintf('Page %d loaded with %d record(s); Total records: %d', $page, $resultCount, $totalCount); $this->comment($comment); } else { $this->comment('No programs found'); } } if ($resultCount < $per_page) { $notLastPage = false; } $options['page'] = $page++; } $count = Program::all()->count(); return $this->info($count . ' programs imported'); }
/** * Execute the console command. * * @return mixed */ public function fire() { $media_id = Config::get("daisycon.media_id"); $sub_id = Config::get("daisycon.sub_id"); $page = 1; $per_page = 50; $notLastPage = true; $options = array('page' => $page, 'per_page' => $per_page, 'media_id' => $media_id, 'placeholder_media_id' => $media_id, 'placeholder_subid' => $sub_id); $this->info('Start importing feeds into the database'); while ($notLastPage) { $APIdata = DaisyconHelper::getRestAPI("productfeeds", $options); if (is_array($APIdata)) { $resultCount = count($APIdata['response']); if ($resultCount > 0) { if ($page == 1) { $this->info('Truncate database table'); Feed::truncate(); } foreach ($APIdata['response'] as $feedinfo) { $feedinfo = (array) $feedinfo; $feedinfo['feed_id'] = $feedinfo['id']; Feed::create($feedinfo); } $totalCount = Feed::all()->count(); $comment = sprintf('Page %d loaded with %d record(s); Total records: %d', $page, $resultCount, $totalCount); $this->comment($comment); } else { return $this->comment('Geen feeds gevonden'); } } if (isset($resultCount) && $resultCount < $per_page) { $notLastPage = false; } $options['page'] = $page++; } $count = Feed::all()->count(); return $this->info($count . ' feeds imported'); }
public static function dataVoorbereiden($data) { $data = parent::fixTransportationType($data); $data = parent::fixBoardingType($data); $data = parent::fixLandcodes($data); $data = parent::fixAccommodationName($data); $data = parent::fixStars($data); $data = parent::fixDescription($data); $data = parent::fixPositions($data); $data = parent::fixDuration($data); // Slug aan het einde laten staan foreach (DaisyconHelper::getDatabaseFields() as $fieldname) { if (str_contains($fieldname, 'slug_')) { $originalFieldName = str_replace('slug_', '', $fieldname); if (!empty($data->{$originalFieldName})) { $data->{$fieldname} = Str::slug($data->{$originalFieldName}); } } // Encoding aanpassen $data->{$fieldname} = html_entity_decode($data->{$fieldname}, ENT_QUOTES, "utf-8"); // nog testen // Trim alle velden $data->{$fieldname} = trim($data->{$fieldname}); // Specialchars aanpassen $regex = '/http:\\/\\/.*/'; if (isset($data->{$fieldname}) && preg_match($regex, $data->{$fieldname})) { $data->{$fieldname} = htmlspecialchars_decode(urldecode($data->{$fieldname})); // sneller dan html_entity_decode @ http://stackoverflow.com/questions/11723641/how-to-decode-the-amp-from-a-url-so-that-header-works-urldecode-not-working // Brakke utm_ arguments vervangen in url's $regex = '/(&utm_\\w+=%\\w+%)&/'; if (preg_match_all($regex, $data->{$fieldname}, $matches)) { $data->{$fieldname} = str_replace($matches[1], '', $data->{$fieldname}); } } } return $data; }