public function addEntryObject($entry, $conf, $feedHistory) { $existingGuids = array_fill_keys($this->listLastGuidsByFeed($entry->feed(), 20), 1); $nb_month_old = max($conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; $eDate = $entry->date(true); if ($feedHistory == -2) { $feedHistory = $conf->keep_history_default; } if (!isset($existingGuids[$entry->guid()]) && ($feedHistory != 0 || $eDate >= $date_min || $entry->isFavorite())) { $values = $entry->toArray(); $useDeclaredDate = empty($existingGuids); $values['id'] = $useDeclaredDate || $eDate < $date_min ? min(time(), $eDate) . uSecString() : uTimeString(); return $this->addEntry($values); } // We don't return Entry object to avoid a research in DB return -1; }
/** * This action actualizes entries from one or several feeds. * * Parameters are: * - id (default: false): Feed ID * - url (default: false): Feed URL * - force (default: false) * If id and url are not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ public function actualizeAction($simplePiePush = null) { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); $entryDAO = FreshRSS_Factory::createEntryDao(); Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); $url = Minz_Request::param('url'); $force = Minz_Request::param('force'); // Create a list of feeds to actualize. // If id is set and valid, corresponding feed is added to the list but // alone in order to automatize further process. $feeds = array(); if ($id || $url) { $feed = $id ? $feedDAO->searchById($id) : $feedDAO->searchByUrl($url); if ($feed) { $feeds[] = $feed; } } else { $feeds = $feedDAO->listFeedsOrderUpdate(FreshRSS_Context::$user_conf->ttl_default); } // Calculate date of oldest entries we accept in DB. $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; // PubSubHubbub support $pubsubhubbubEnabledGeneral = FreshRSS_Context::$system_conf->pubsubhubbub_enabled; $pshbMinAge = time() - 3600 * 24; //TODO: Make a configuration. $updated_feeds = 0; $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; foreach ($feeds as $feed) { $url = $feed->url(); //For detection of HTTP 301 $pubSubHubbubEnabled = $pubsubhubbubEnabledGeneral && $feed->pubSubHubbubEnabled(); if (!$simplePiePush && !$id && $pubSubHubbubEnabled && $feed->lastUpdate() > $pshbMinAge) { //$text = 'Skip pull of feed using PubSubHubbub: ' . $url; //Minz_Log::debug($text); //file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND); continue; //When PubSubHubbub is used, do not pull refresh so often } if (!$feed->lock()) { Minz_Log::notice('Feed already being actualized: ' . $feed->url()); continue; } try { if ($simplePiePush) { $feed->loadEntries($simplePiePush); //Used by PubSubHubbub } else { $feed->load(false); } } catch (FreshRSS_Feed_Exception $e) { Minz_Log::warning($e->getMessage()); $feedDAO->updateLastUpdate($feed->id(), true); $feed->unlock(); continue; } $feed_history = $feed->keepHistory(); if ($feed_history == -2) { // TODO: -2 must be a constant! // -2 means we take the default value from configuration $feed_history = FreshRSS_Context::$user_conf->keep_history_default; } // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { $newGuids = array(); foreach ($entries as $entry) { $newGuids[] = $entry->guid(); } // For this feed, check existing GUIDs already in database. $existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids); unset($newGuids); $oldGuids = array(); // Add entries in database if possible. foreach ($entries as $entry) { $entry_date = $entry->date(true); if (isset($existingHashForGuids[$entry->guid()])) { $existingHash = $existingHashForGuids[$entry->guid()]; if (strcasecmp($existingHash, $entry->hash()) === 0 || $existingHash === '00000000000000000000000000000000') { //This entry already exists and is unchanged. TODO: Remove the test with the zero'ed hash in FreshRSS v1.3 $oldGuids[] = $entry->guid(); } else { //This entry already exists but has been updated Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() . ', old hash ' . $existingHash . ', new hash ' . $entry->hash()); //TODO: Make an updated/is_read policy by feed, in addition to the global one. $entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null); //Change is_read according to policy. if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $entryDAO->updateEntry($entry->toArray()); } } elseif ($feed_history == 0 && $entry_date < $date_min) { // This entry should not be added considering configuration and date. $oldGuids[] = $entry->guid(); } else { if ($entry_date < $date_min) { $id = min(time(), $entry_date) . uSecString(); $entry->_isRead(true); //Old article that was not in database. Probably an error, so mark as read } else { $id = uTimeString(); $entry->_isRead($is_read); } $entry->_id($id); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if ($entry === null) { // An extension has returned a null value, there is nothing to insert. continue; } if ($pubSubHubbubEnabled && !$simplePiePush) { //We use push, but have discovered an article by pull! $text = 'An article was discovered by pull although we use PubSubHubbub!: Feed ' . $url . ' GUID ' . $entry->guid(); file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND); Minz_Log::warning($text); $pubSubHubbubEnabled = false; $feed->pubSubHubbubError(true); } if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $entryDAO->addEntry($entry->toArray()); } } $entryDAO->updateLastSeen($feed->id(), $oldGuids); } if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10)); if ($nb > 0) { Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url() . ']'); } } $feedDAO->updateLastUpdate($feed->id(), 0, $entryDAO->hasTransaction()); if ($entryDAO->hasTransaction()) { $entryDAO->commit(); } if ($feed->hubUrl() && $feed->selfUrl()) { //selfUrl has priority for PubSubHubbub if ($feed->selfUrl() !== $url) { //https://code.google.com/p/pubsubhubbub/wiki/MovingFeedsOrChangingHubs $selfUrl = checkUrl($feed->selfUrl()); if ($selfUrl) { Minz_Log::debug('PubSubHubbub unsubscribe ' . $feed->url()); if (!$feed->pubSubHubbubSubscribe(false)) { //Unsubscribe Minz_Log::warning('Error while PubSubHubbub unsubscribing from ' . $feed->url()); } $feed->_url($selfUrl, false); Minz_Log::notice('Feed ' . $url . ' canonical address moved to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } } } elseif ($feed->url() !== $url) { // HTTP 301 Moved Permanently Minz_Log::notice('Feed ' . $url . ' moved permanently to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } $feed->faviconPrepare(); if ($pubsubhubbubEnabledGeneral && $feed->pubSubHubbubPrepare()) { Minz_Log::notice('PubSubHubbub subscribe ' . $feed->url()); if (!$feed->pubSubHubbubSubscribe(true)) { //Subscribe Minz_Log::warning('Error while PubSubHubbub subscribing to ' . $feed->url()); } } $feed->unlock(); $updated_feeds++; unset($feed); // No more than 10 feeds unless $force is true to avoid overloading // the server. if ($updated_feeds >= 10 && !$force) { break; } } if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since // there are several parallel requests, we should return that there // are several updated feeds. $notif = array('type' => 'good', 'content' => _t('feedback.sub.feed.actualizeds')); Minz_Session::_param('notification', $notif); // No layout in ajax request. $this->view->_useLayout(false); } else { // Redirect to the main page with correct notification. if ($updated_feeds === 1) { $feed = reset($feeds); Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array('params' => array('get' => 'f_' . $feed->id()))); } elseif ($updated_feeds > 1) { Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); } else { Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); } } return $updated_feeds; }
/** * This method import a JSON-based file (Google Reader format). * * @param string $article_file the JSON file content. * @param boolean $starred true if articles from the file must be starred. * @return boolean true if an error occured, false else. */ private function importJson($article_file, $starred = false) { $article_object = json_decode($article_file, true); if (is_null($article_object)) { Minz_Log::warning('Try to import a non-JSON file'); return true; } $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; $google_compliant = strpos($article_object['id'], 'com.google') !== false; $error = false; $article_to_feed = array(); $nb_feeds = count($this->feedDAO->listFeeds()); $limits = FreshRSS_Context::$system_conf->limits; // First, we check feeds of articles are in DB (and add them if needed). foreach ($article_object['items'] as $item) { $key = $google_compliant ? 'htmlUrl' : 'feedUrl'; $feed = new FreshRSS_Feed($item['origin'][$key]); $feed = $this->feedDAO->searchByUrl($feed->url()); if (is_null($feed)) { // Feed does not exist in DB,we should to try to add it. if ($nb_feeds >= $limits['max_feeds']) { // Oops, no more place! Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds'])); } else { $feed = $this->addFeedJson($item['origin'], $google_compliant); } if (is_null($feed)) { // Still null? It means something went wrong. $error = true; } else { // Nice! Increase the counter. $nb_feeds += 1; } } if (!is_null($feed)) { $article_to_feed[$item['id']] = $feed->id(); } } // Then, articles are imported. $prepared_statement = $this->entryDAO->addEntryPrepare(); $this->entryDAO->beginTransaction(); foreach ($article_object['items'] as $item) { if (!isset($article_to_feed[$item['id']])) { // Related feed does not exist for this entry, do nothing. continue; } $feed_id = $article_to_feed[$item['id']]; $author = isset($item['author']) ? $item['author'] : ''; $key_content = $google_compliant && !isset($item['content']) ? 'summary' : 'content'; $tags = $item['categories']; if ($google_compliant) { // Remove tags containing "/state/com.google" which are useless. $tags = array_filter($tags, function ($var) { return strpos($var, '/state/com.google') === false; }); } $entry = new FreshRSS_Entry($feed_id, $item['id'], $item['title'], $author, $item[$key_content]['content'], $item['alternate'][0]['href'], $item['published'], $is_read, $starred); $entry->_id(min(time(), $entry->date(true)) . uSecString()); $entry->_tags($tags); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if (is_null($entry)) { // An extension has returned a null value, there is nothing to insert. continue; } $values = $entry->toArray(); $id = $this->entryDAO->addEntry($values, $prepared_statement); if (!$error && $id === false) { $error = true; } } $this->entryDAO->commit(); return $error; }
public function actualizeAction() { @set_time_limit(300); $feedDAO = new FreshRSS_FeedDAO(); $entryDAO = new FreshRSS_EntryDAO(); Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); $force = Minz_Request::param('force', false); // on créé la liste des flux à mettre à actualiser // si on veut mettre un flux à jour spécifiquement, on le met // dans la liste, mais seul (permet d'automatiser le traitement) $feeds = array(); if ($id) { $feed = $feedDAO->searchById($id); if ($feed) { $feeds = array($feed); } } else { $feeds = $feedDAO->listFeedsOrderUpdate(); } // on calcule la date des articles les plus anciens qu'on accepte $nb_month_old = max($this->view->conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; $i = 0; $flux_update = 0; $is_read = $this->view->conf->mark_when['reception'] ? 1 : 0; foreach ($feeds as $feed) { if (!$feed->lock()) { Minz_Log::record('Feed already being actualized: ' . $feed->url(), Minz_Log::NOTICE); continue; } try { $url = $feed->url(); $feedHistory = $feed->keepHistory(); $feed->load(false); $entries = array_reverse($feed->entries()); //We want chronological order and SimplePie uses reverse order $hasTransaction = false; if (count($entries) > 0) { //For this feed, check last n entry GUIDs already in database $existingGuids = array_fill_keys($entryDAO->listLastGuidsByFeed($feed->id(), count($entries) + 10), 1); $useDeclaredDate = empty($existingGuids); if ($feedHistory == -2) { //default $feedHistory = $this->view->conf->keep_history_default; } $hasTransaction = true; $feedDAO->beginTransaction(); // On ne vérifie pas strictement que l'article n'est pas déjà en BDD // La BDD refusera l'ajout car (id_feed, guid) doit être unique foreach ($entries as $entry) { $eDate = $entry->date(true); if (!isset($existingGuids[$entry->guid()]) && ($feedHistory != 0 || $eDate >= $date_min)) { $values = $entry->toArray(); //Use declared date at first import, otherwise use discovery date $values['id'] = $useDeclaredDate || $eDate < $date_min ? min(time(), $eDate) . uSecString() : uTimeString(); $values['is_read'] = $is_read; $entryDAO->addEntry($values); } } } if ($feedHistory >= 0 && rand(0, 30) === 1) { if (!$hasTransaction) { $feedDAO->beginTransaction(); } $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, max($feedHistory, count($entries) + 10)); if ($nb > 0) { Minz_Log::record($nb . ' old entries cleaned in feed [' . $feed->url() . ']', Minz_Log::DEBUG); } } // on indique que le flux vient d'être mis à jour en BDD $feedDAO->updateLastUpdate($feed->id(), 0, $hasTransaction); if ($hasTransaction) { $feedDAO->commit(); } $flux_update++; if ($feed->url() !== $url) { //URL has changed (auto-discovery) $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } } catch (FreshRSS_Feed_Exception $e) { Minz_Log::record($e->getMessage(), Minz_Log::NOTICE); $feedDAO->updateLastUpdate($feed->id(), 1); } $feed->faviconPrepare(); $feed->unlock(); unset($feed); // On arrête à 10 flux pour ne pas surcharger le serveur // sauf si le paramètre $force est à vrai $i++; if ($i >= 10 && !$force) { break; } } $url = array(); if ($flux_update === 1) { // on a mis un seul flux à jour $feed = reset($feeds); $notif = array('type' => 'good', 'content' => Minz_Translate::t('feed_actualized', $feed->name())); } elseif ($flux_update > 1) { // plusieurs flux on été mis à jour $notif = array('type' => 'good', 'content' => Minz_Translate::t('n_feeds_actualized', $flux_update)); } else { // aucun flux n'a été mis à jour, oups $notif = array('type' => 'good', 'content' => Minz_Translate::t('no_feed_to_refresh')); } if ($i === 1) { // Si on a voulu mettre à jour qu'un flux // on filtre l'affichage par ce flux $feed = reset($feeds); $url['params'] = array('get' => 'f_' . $feed->id()); } if (Minz_Request::param('ajax', 0) === 0) { Minz_Session::_param('notification', $notif); Minz_Request::forward($url, true); } else { // Une requête Ajax met un seul flux à jour. // Comme en principe plusieurs requêtes ont lieu, // on indique que "plusieurs flux ont été mis à jour". // Cela permet d'avoir une notification plus proche du // ressenti utilisateur $notif = array('type' => 'good', 'content' => Minz_Translate::t('feeds_actualized')); Minz_Session::_param('notification', $notif); // et on désactive le layout car ne sert à rien $this->view->_useLayout(false); } }
/** * This action actualizes entries from one or several feeds. * * Parameters are: * - id (default: false) * - force (default: false) * If id is not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ public function actualizeAction() { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); $entryDAO = FreshRSS_Factory::createEntryDao(); Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); $force = Minz_Request::param('force'); // Create a list of feeds to actualize. // If id is set and valid, corresponding feed is added to the list but // alone in order to automatize further process. $feeds = array(); if ($id) { $feed = $feedDAO->searchById($id); if ($feed) { $feeds[] = $feed; } } else { $feeds = $feedDAO->listFeedsOrderUpdate(FreshRSS_Context::$user_conf->ttl_default); } // Calculate date of oldest entries we accept in DB. $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; $updated_feeds = 0; $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; foreach ($feeds as $feed) { if (!$feed->lock()) { Minz_Log::notice('Feed already being actualized: ' . $feed->url()); continue; } try { // Load entries $feed->load(false); } catch (FreshRSS_Feed_Exception $e) { Minz_Log::notice($e->getMessage()); $feedDAO->updateLastUpdate($feed->id(), 1); $feed->unlock(); continue; } $url = $feed->url(); $feed_history = $feed->keepHistory(); if ($feed_history == -2) { // TODO: -2 must be a constant! // -2 means we take the default value from configuration $feed_history = FreshRSS_Context::$user_conf->keep_history_default; } // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { // For this feed, check last n entry GUIDs already in database. $existing_guids = array_fill_keys($entryDAO->listLastGuidsByFeed($feed->id(), count($entries) + 10), 1); $use_declared_date = empty($existing_guids); // Add entries in database if possible. $prepared_statement = $entryDAO->addEntryPrepare(); $feedDAO->beginTransaction(); foreach ($entries as $entry) { $entry_date = $entry->date(true); if (isset($existing_guids[$entry->guid()]) || $feed_history == 0 && $entry_date < $date_min) { // This entry already exists in DB or should not be added // considering configuration and date. continue; } $id = uTimeString(); if ($use_declared_date || $entry_date < $date_min) { // Use declared date at first import. $id = min(time(), $entry_date) . uSecString(); } $entry->_id($id); $entry->_isRead($is_read); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if (is_null($entry)) { // An extension has returned a null value, there is nothing to insert. continue; } $values = $entry->toArray(); $entryDAO->addEntry($values, $prepared_statement); } } if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. if (!$feedDAO->hasTransaction()) { $feedDAO->beginTransaction(); } $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10)); if ($nb > 0) { Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url() . ']'); } } $feedDAO->updateLastUpdate($feed->id(), 0, $feedDAO->hasTransaction()); if ($feedDAO->hasTransaction()) { $feedDAO->commit(); } if ($feed->url() !== $url) { // HTTP 301 Moved Permanently Minz_Log::notice('Feed ' . $url . ' moved permanently to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } $feed->faviconPrepare(); $feed->unlock(); $updated_feeds++; unset($feed); // No more than 10 feeds unless $force is true to avoid overloading // the server. if ($updated_feeds >= 10 && !$force) { break; } } if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since // there are several parallel requests, we should return that there // are several updated feeds. $notif = array('type' => 'good', 'content' => _t('feedback.sub.feed.actualizeds')); Minz_Session::_param('notification', $notif); // No layout in ajax request. $this->view->_useLayout(false); return; } // Redirect to the main page with correct notification. if ($updated_feeds === 1) { $feed = reset($feeds); Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array('params' => array('get' => 'f_' . $feed->id()))); } elseif ($updated_feeds > 1) { Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); } else { Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); } }