public function applyAction() { if (!file_exists(UPDATE_FILENAME) || !is_writable(FRESHRSS_PATH)) { Minz_Request::forward(array('c' => 'update'), true); } require UPDATE_FILENAME; if (Minz_Request::param('post_conf', false)) { $res = do_post_update(); Minz_ExtensionManager::callHook('post_update'); if ($res === true) { @unlink(UPDATE_FILENAME); @file_put_contents(join_path(DATA_PATH, 'last_update.txt'), ''); Minz_Request::good(_t('feedback.update.finished')); } else { Minz_Request::bad(_t('feedback.update.error', $res), array('c' => 'update', 'a' => 'index')); } } if (Minz_Request::isPost()) { save_info_update(); } if (!need_info_update()) { $res = apply_update(); if ($res === true) { Minz_Request::forward(array('c' => 'update', 'a' => 'apply', 'params' => array('post_conf' => true)), true); } else { Minz_Request::bad(_t('feedback.update.error', $res), array('c' => 'update', 'a' => 'index')); } } }
/** * This action actualizes entries from one or several feeds. * * Parameters are: * - id (default: false): Feed ID * - url (default: false): Feed URL * - force (default: false) * If id and url are not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ public function actualizeAction($simplePiePush = null) { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); $entryDAO = FreshRSS_Factory::createEntryDao(); Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); $url = Minz_Request::param('url'); $force = Minz_Request::param('force'); // Create a list of feeds to actualize. // If id is set and valid, corresponding feed is added to the list but // alone in order to automatize further process. $feeds = array(); if ($id || $url) { $feed = $id ? $feedDAO->searchById($id) : $feedDAO->searchByUrl($url); if ($feed) { $feeds[] = $feed; } } else { $feeds = $feedDAO->listFeedsOrderUpdate(FreshRSS_Context::$user_conf->ttl_default); } // Calculate date of oldest entries we accept in DB. $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; // PubSubHubbub support $pubsubhubbubEnabledGeneral = FreshRSS_Context::$system_conf->pubsubhubbub_enabled; $pshbMinAge = time() - 3600 * 24; //TODO: Make a configuration. $updated_feeds = 0; $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; foreach ($feeds as $feed) { $url = $feed->url(); //For detection of HTTP 301 $pubSubHubbubEnabled = $pubsubhubbubEnabledGeneral && $feed->pubSubHubbubEnabled(); if (!$simplePiePush && !$id && $pubSubHubbubEnabled && $feed->lastUpdate() > $pshbMinAge) { //$text = 'Skip pull of feed using PubSubHubbub: ' . $url; //Minz_Log::debug($text); //file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND); continue; //When PubSubHubbub is used, do not pull refresh so often } if (!$feed->lock()) { Minz_Log::notice('Feed already being actualized: ' . $feed->url()); continue; } try { if ($simplePiePush) { $feed->loadEntries($simplePiePush); //Used by PubSubHubbub } else { $feed->load(false); } } catch (FreshRSS_Feed_Exception $e) { Minz_Log::warning($e->getMessage()); $feedDAO->updateLastUpdate($feed->id(), true); $feed->unlock(); continue; } $feed_history = $feed->keepHistory(); if ($feed_history == -2) { // TODO: -2 must be a constant! // -2 means we take the default value from configuration $feed_history = FreshRSS_Context::$user_conf->keep_history_default; } // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { $newGuids = array(); foreach ($entries as $entry) { $newGuids[] = $entry->guid(); } // For this feed, check existing GUIDs already in database. $existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids); unset($newGuids); $oldGuids = array(); // Add entries in database if possible. foreach ($entries as $entry) { $entry_date = $entry->date(true); if (isset($existingHashForGuids[$entry->guid()])) { $existingHash = $existingHashForGuids[$entry->guid()]; if (strcasecmp($existingHash, $entry->hash()) === 0 || $existingHash === '00000000000000000000000000000000') { //This entry already exists and is unchanged. TODO: Remove the test with the zero'ed hash in FreshRSS v1.3 $oldGuids[] = $entry->guid(); } else { //This entry already exists but has been updated Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() . ', old hash ' . $existingHash . ', new hash ' . $entry->hash()); //TODO: Make an updated/is_read policy by feed, in addition to the global one. $entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null); //Change is_read according to policy. if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $entryDAO->updateEntry($entry->toArray()); } } elseif ($feed_history == 0 && $entry_date < $date_min) { // This entry should not be added considering configuration and date. $oldGuids[] = $entry->guid(); } else { if ($entry_date < $date_min) { $id = min(time(), $entry_date) . uSecString(); $entry->_isRead(true); //Old article that was not in database. Probably an error, so mark as read } else { $id = uTimeString(); $entry->_isRead($is_read); } $entry->_id($id); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if ($entry === null) { // An extension has returned a null value, there is nothing to insert. continue; } if ($pubSubHubbubEnabled && !$simplePiePush) { //We use push, but have discovered an article by pull! $text = 'An article was discovered by pull although we use PubSubHubbub!: Feed ' . $url . ' GUID ' . $entry->guid(); file_put_contents(USERS_PATH . '/_/log_pshb.txt', date('c') . "\t" . $text . "\n", FILE_APPEND); Minz_Log::warning($text); $pubSubHubbubEnabled = false; $feed->pubSubHubbubError(true); } if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $entryDAO->addEntry($entry->toArray()); } } $entryDAO->updateLastSeen($feed->id(), $oldGuids); } if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. if (!$entryDAO->hasTransaction()) { $entryDAO->beginTransaction(); } $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10)); if ($nb > 0) { Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url() . ']'); } } $feedDAO->updateLastUpdate($feed->id(), 0, $entryDAO->hasTransaction()); if ($entryDAO->hasTransaction()) { $entryDAO->commit(); } if ($feed->hubUrl() && $feed->selfUrl()) { //selfUrl has priority for PubSubHubbub if ($feed->selfUrl() !== $url) { //https://code.google.com/p/pubsubhubbub/wiki/MovingFeedsOrChangingHubs $selfUrl = checkUrl($feed->selfUrl()); if ($selfUrl) { Minz_Log::debug('PubSubHubbub unsubscribe ' . $feed->url()); if (!$feed->pubSubHubbubSubscribe(false)) { //Unsubscribe Minz_Log::warning('Error while PubSubHubbub unsubscribing from ' . $feed->url()); } $feed->_url($selfUrl, false); Minz_Log::notice('Feed ' . $url . ' canonical address moved to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } } } elseif ($feed->url() !== $url) { // HTTP 301 Moved Permanently Minz_Log::notice('Feed ' . $url . ' moved permanently to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } $feed->faviconPrepare(); if ($pubsubhubbubEnabledGeneral && $feed->pubSubHubbubPrepare()) { Minz_Log::notice('PubSubHubbub subscribe ' . $feed->url()); if (!$feed->pubSubHubbubSubscribe(true)) { //Subscribe Minz_Log::warning('Error while PubSubHubbub subscribing to ' . $feed->url()); } } $feed->unlock(); $updated_feeds++; unset($feed); // No more than 10 feeds unless $force is true to avoid overloading // the server. if ($updated_feeds >= 10 && !$force) { break; } } if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since // there are several parallel requests, we should return that there // are several updated feeds. $notif = array('type' => 'good', 'content' => _t('feedback.sub.feed.actualizeds')); Minz_Session::_param('notification', $notif); // No layout in ajax request. $this->view->_useLayout(false); } else { // Redirect to the main page with correct notification. if ($updated_feeds === 1) { $feed = reset($feeds); Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array('params' => array('get' => 'f_' . $feed->id()))); } elseif ($updated_feeds > 1) { Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); } else { Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); } } return $updated_feeds; }
/** * This method import a JSON-based feed (Google Reader format). * * @param array $origin represents a feed. * @param boolean $google_compliant takes care of some specific values if true. * @return FreshRSS_Feed if feed is in database at the end of the process, * else null. */ private function addFeedJson($origin, $google_compliant) { $default_cat = $this->catDAO->getDefault(); $return = null; $key = $google_compliant ? 'htmlUrl' : 'feedUrl'; $url = $origin[$key]; $name = $origin['title']; $website = $origin['htmlUrl']; try { // Create a Feed object and add it in database. $feed = new FreshRSS_Feed($url); $feed->_category($default_cat->id()); $feed->_name($name); $feed->_website($website); // Call the extension hook $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed); if (!is_null($feed)) { // addFeedObject checks if feed is already in DB so nothing else to // check here. $id = $this->feedDAO->addFeedObject($feed); if ($id !== false) { $feed->_id($id); $return = $feed; } } } catch (FreshRSS_Feed_Exception $e) { Minz_Log::warning($e->getMessage()); } return $return; }
/** * This action actualizes entries from one or several feeds. * * Parameters are: * - id (default: false) * - force (default: false) * If id is not specified, all the feeds are actualized. But if force is * false, process stops at 10 feeds to avoid time execution problem. */ public function actualizeAction() { @set_time_limit(300); $feedDAO = FreshRSS_Factory::createFeedDao(); $entryDAO = FreshRSS_Factory::createEntryDao(); Minz_Session::_param('actualize_feeds', false); $id = Minz_Request::param('id'); $force = Minz_Request::param('force'); // Create a list of feeds to actualize. // If id is set and valid, corresponding feed is added to the list but // alone in order to automatize further process. $feeds = array(); if ($id) { $feed = $feedDAO->searchById($id); if ($feed) { $feeds[] = $feed; } } else { $feeds = $feedDAO->listFeedsOrderUpdate(FreshRSS_Context::$user_conf->ttl_default); } // Calculate date of oldest entries we accept in DB. $nb_month_old = max(FreshRSS_Context::$user_conf->old_entries, 1); $date_min = time() - 3600 * 24 * 30 * $nb_month_old; $updated_feeds = 0; $is_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0; foreach ($feeds as $feed) { if (!$feed->lock()) { Minz_Log::notice('Feed already being actualized: ' . $feed->url()); continue; } try { // Load entries $feed->load(false); } catch (FreshRSS_Feed_Exception $e) { Minz_Log::notice($e->getMessage()); $feedDAO->updateLastUpdate($feed->id(), 1); $feed->unlock(); continue; } $url = $feed->url(); $feed_history = $feed->keepHistory(); if ($feed_history == -2) { // TODO: -2 must be a constant! // -2 means we take the default value from configuration $feed_history = FreshRSS_Context::$user_conf->keep_history_default; } // We want chronological order and SimplePie uses reverse order. $entries = array_reverse($feed->entries()); if (count($entries) > 0) { // For this feed, check last n entry GUIDs already in database. $existing_guids = array_fill_keys($entryDAO->listLastGuidsByFeed($feed->id(), count($entries) + 10), 1); $use_declared_date = empty($existing_guids); // Add entries in database if possible. $prepared_statement = $entryDAO->addEntryPrepare(); $feedDAO->beginTransaction(); foreach ($entries as $entry) { $entry_date = $entry->date(true); if (isset($existing_guids[$entry->guid()]) || $feed_history == 0 && $entry_date < $date_min) { // This entry already exists in DB or should not be added // considering configuration and date. continue; } $id = uTimeString(); if ($use_declared_date || $entry_date < $date_min) { // Use declared date at first import. $id = min(time(), $entry_date) . uSecString(); } $entry->_id($id); $entry->_isRead($is_read); $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry); if (is_null($entry)) { // An extension has returned a null value, there is nothing to insert. continue; } $values = $entry->toArray(); $entryDAO->addEntry($values, $prepared_statement); } } if ($feed_history >= 0 && rand(0, 30) === 1) { // TODO: move this function in web cron when available (see entry::purge) // Remove old entries once in 30. if (!$feedDAO->hasTransaction()) { $feedDAO->beginTransaction(); } $nb = $feedDAO->cleanOldEntries($feed->id(), $date_min, max($feed_history, count($entries) + 10)); if ($nb > 0) { Minz_Log::debug($nb . ' old entries cleaned in feed [' . $feed->url() . ']'); } } $feedDAO->updateLastUpdate($feed->id(), 0, $feedDAO->hasTransaction()); if ($feedDAO->hasTransaction()) { $feedDAO->commit(); } if ($feed->url() !== $url) { // HTTP 301 Moved Permanently Minz_Log::notice('Feed ' . $url . ' moved permanently to ' . $feed->url()); $feedDAO->updateFeed($feed->id(), array('url' => $feed->url())); } $feed->faviconPrepare(); $feed->unlock(); $updated_feeds++; unset($feed); // No more than 10 feeds unless $force is true to avoid overloading // the server. if ($updated_feeds >= 10 && !$force) { break; } } if (Minz_Request::param('ajax')) { // Most of the time, ajax request is for only one feed. But since // there are several parallel requests, we should return that there // are several updated feeds. $notif = array('type' => 'good', 'content' => _t('feedback.sub.feed.actualizeds')); Minz_Session::_param('notification', $notif); // No layout in ajax request. $this->view->_useLayout(false); return; } // Redirect to the main page with correct notification. if ($updated_feeds === 1) { $feed = reset($feeds); Minz_Request::good(_t('feedback.sub.feed.actualized', $feed->name()), array('params' => array('get' => 'f_' . $feed->id()))); } elseif ($updated_feeds > 1) { Minz_Request::good(_t('feedback.sub.feed.n_actualized', $updated_feeds), array()); } else { Minz_Request::good(_t('feedback.sub.feed.no_refresh'), array()); } }