/**
* Additional validation includes checking URL and tags
*/
public function validation($data, $files)
{
global $CFG;
$errors = parent::validation($data, $files);
require_once $CFG->libdir . '/simplepie/moodle_simplepie.php';
$rss = new moodle_simplepie();
$rssfile = $rss->registry->create('File', array($data['url']));
$filetest = $rss->registry->create('Locator', array($rssfile));
if (!$filetest->is_feed($rssfile)) {
$errors['url'] = get_string('feedisinvalid', 'blog');
} else {
$rss->set_feed_url($data['url']);
if (!$rss->init()) {
$errors['url'] = get_string('emptyrssfeed', 'blog');
}
}
return $errors;
}
/**
* cron - goes through all feeds and retrieves them with the cache
* duration set to 0 in order to force the retrieval of the item and
* refresh the cache
*
* @return boolean true if all feeds were retrieved succesfully
*/
function cron()
{
global $CFG, $DB;
require_once $CFG->libdir . '/simplepie/moodle_simplepie.php';
// We are going to measure execution times
$starttime = microtime();
// And we have one initial $status
$status = true;
// Fetch all site feeds.
$rs = $DB->get_recordset('block_rss_client');
$counter = 0;
mtrace('');
foreach ($rs as $rec) {
mtrace(' ' . $rec->url . ' ', '');
// Fetch the rss feed, using standard simplepie caching
// so feeds will be renewed only if cache has expired
@set_time_limit(60);
$feed = new moodle_simplepie();
// set timeout for longer than normal to be agressive at
// fetching feeds if possible..
$feed->set_timeout(40);
$feed->set_cache_duration(0);
$feed->set_feed_url($rec->url);
$feed->init();
if ($feed->error()) {
mtrace('error');
mtrace('SimplePie failed with error:' . $feed->error());
$status = false;
} else {
mtrace('ok');
}
$counter++;
}
$rs->close();
// Show times
mtrace($counter . ' feeds refreshed (took ' . microtime_diff($starttime, microtime()) . ' seconds)');
// And return $status
return $status;
}
/**
* Given a record in the {blog_external} table, checks the blog's URL
* for new entries not yet copied into Moodle.
* Also attempts to identify and remove deleted blog entries
*
* @param object $externalblog
* @return boolean False if the Feed is invalid
*/
function blog_sync_external_entries($externalblog)
{
global $CFG, $DB;
require_once $CFG->libdir . '/simplepie/moodle_simplepie.php';
$rss = new moodle_simplepie();
$rssfile = $rss->registry->create('File', array($externalblog->url));
$filetest = $rss->registry->create('Locator', array($rssfile));
if (!$filetest->is_feed($rssfile)) {
$externalblog->failedlastsync = 1;
$DB->update_record('blog_external', $externalblog);
return false;
} else {
if (!empty($externalblog->failedlastsync)) {
$externalblog->failedlastsync = 0;
$DB->update_record('blog_external', $externalblog);
}
}
$rss->set_feed_url($externalblog->url);
$rss->init();
if (empty($rss->data)) {
return null;
}
//used to identify blog posts that have been deleted from the source feed
$oldesttimestamp = null;
$uniquehashes = array();
foreach ($rss->get_items() as $entry) {
// If filtertags are defined, use them to filter the entries by RSS category
if (!empty($externalblog->filtertags)) {
$containsfiltertag = false;
$categories = $entry->get_categories();
$filtertags = explode(',', $externalblog->filtertags);
$filtertags = array_map('trim', $filtertags);
$filtertags = array_map('strtolower', $filtertags);
foreach ($categories as $category) {
if (in_array(trim(strtolower($category->term)), $filtertags)) {
$containsfiltertag = true;
}
}
if (!$containsfiltertag) {
continue;
}
}
$uniquehashes[] = $entry->get_permalink();
$newentry = new stdClass();
$newentry->userid = $externalblog->userid;
$newentry->module = 'blog_external';
$newentry->content = $externalblog->id;
$newentry->uniquehash = $entry->get_permalink();
$newentry->publishstate = 'site';
$newentry->format = FORMAT_HTML;
// Clean subject of html, just in case
$newentry->subject = clean_param($entry->get_title(), PARAM_TEXT);
// Observe 128 max chars in DB
// TODO: +1 to raise this to 255
if (core_text::strlen($newentry->subject) > 128) {
$newentry->subject = core_text::substr($newentry->subject, 0, 125) . '...';
}
$newentry->summary = $entry->get_description();
//used to decide whether to insert or update
//uses enty permalink plus creation date if available
$existingpostconditions = array('uniquehash' => $entry->get_permalink());
//our DB doesnt allow null creation or modified timestamps so check the external blog supplied one
$entrydate = $entry->get_date('U');
if (!empty($entrydate)) {
$existingpostconditions['created'] = $entrydate;
}
//the post ID or false if post not found in DB
$postid = $DB->get_field('post', 'id', $existingpostconditions);
$timestamp = null;
if (empty($entrydate)) {
$timestamp = time();
} else {
$timestamp = $entrydate;
}
//only set created if its a new post so we retain the original creation timestamp if the post is edited
if ($postid === false) {
$newentry->created = $timestamp;
}
$newentry->lastmodified = $timestamp;
if (empty($oldesttimestamp) || $timestamp < $oldesttimestamp) {
//found an older post
$oldesttimestamp = $timestamp;
}
if (core_text::strlen($newentry->uniquehash) > 255) {
// The URL for this item is too long for the field. Rather than add
// the entry without the link we will skip straight over it.
// RSS spec says recommended length 500, we use 255.
debugging('External blog entry skipped because of oversized URL', DEBUG_DEVELOPER);
continue;
}
if ($postid === false) {
$id = $DB->insert_record('post', $newentry);
// Set tags
if ($tags = tag_get_tags_array('blog_external', $externalblog->id)) {
tag_set('post', $id, $tags);
}
} else {
$newentry->id = $postid;
$DB->update_record('post', $newentry);
}
}
// Look at the posts we have in the database to check if any of them have been deleted from the feed.
// Only checking posts within the time frame returned by the rss feed. Older items may have been deleted or
// may just not be returned anymore. We can't tell the difference so we leave older posts alone.
$sql = "SELECT id, uniquehash\n FROM {post}\n WHERE module = 'blog_external'\n AND " . $DB->sql_compare_text('content') . " = " . $DB->sql_compare_text(':blogid') . "\n AND created > :ts";
$dbposts = $DB->get_records_sql($sql, array('blogid' => $externalblog->id, 'ts' => $oldesttimestamp));
$todelete = array();
foreach ($dbposts as $dbpost) {
if (!in_array($dbpost->uniquehash, $uniquehashes)) {
$todelete[] = $dbpost->id;
}
}
$DB->delete_records_list('post', 'id', $todelete);
$DB->update_record('blog_external', array('id' => $externalblog->id, 'timefetched' => time()));
}
/**
* Autodiscovers a feed url from a given url, to be used by the formslibs
* filter function
*
* Uses simplepie with autodiscovery set to maximum level to try and find
* a feed to subscribe to.
* See: http://simplepie.org/wiki/reference/simplepie/set_autodiscovery_level
*
* @param string URL to autodiscover a url
* @return string URL of feed or original url if none found
*/
public static function autodiscover_feed_url($url)
{
$rss = new moodle_simplepie();
$rss->set_feed_url($url);
$rss->set_autodiscovery_level(SIMPLEPIE_LOCATOR_ALL);
// When autodiscovering an RSS feed, simplepie will try lots of
// rss links on a page, so set the timeout high
$rss->set_timeout(20);
$rss->init();
if ($rss->error()) {
return $url;
}
return $rss->subscribe_url();
}
/**
* Fetch the activity stream from the tracker and register the activity items
*
* @param int $before optionally filter activity that happened before this timestamp
* @param int $after optionally filter activity that happened after this timestamp
* @internal
*/
function dev_register_tracker_activity($before = null, $after = null)
{
global $DB;
$filters = array();
$filters[] = 'streams=' . rawurlencode('key+IS+MDL+MDLQA+MDLSITE+MOBILE');
if (!is_null($before) and !is_null($after)) {
$filters[] = 'streams=' . rawurlencode('update-date+BETWEEN+' . $after * 1000 . '+' . $before * 1000);
} else {
if (!is_null($before)) {
$filters[] = 'streams=' . rawurlencode('update-date+BEFORE+' . $before * 1000);
} else {
if (!is_null($after)) {
$filters[] = 'streams=' . rawurlencode('update-date+AFTER+' . $after * 1000);
}
}
}
$url = 'https://tracker.moodle.org/activity?' . implode('&', $filters);
if (!is_null($after)) {
fputs(STDOUT, date("Y-m-d H:i:s", $after));
} else {
fputs(STDOUT, '*');
}
fputs(STDOUT, ' - ');
if (!is_null($before)) {
fputs(STDOUT, date("Y-m-d H:i:s", $before));
} else {
fputs(STDOUT, '*');
}
$feed = new moodle_simplepie();
$feed->set_timeout(10);
$feed->set_feed_url($url);
$feed->init();
if ($error = $feed->error()) {
fputs(STDERR, $error . PHP_EOL);
exit(1);
}
$fetched = 0;
$created = 0;
foreach ($feed->get_items() as $item) {
$fetched++;
$activity = new stdClass();
$activity->uuid = $item->get_id();
$activity->title = $item->get_title();
$activity->timecreated = $item->get_date('U');
$activity->link = $item->get_link();
if ($tmp = $item->get_category()) {
$activity->category = $tmp->get_term();
}
if ($tmp = $item->get_author()) {
$activity->personfullname = $tmp->get_name();
$activity->personemail = $tmp->get_email();
$activity->personlink = $tmp->get_link();
}
if (!$DB->record_exists('dev_tracker_activities', array('uuid' => $activity->uuid))) {
$DB->insert_record('dev_tracker_activities', $activity, false, true);
$created++;
}
}
fputs(STDOUT, sprintf(" %d %d %s\n", $fetched, $created, rawurldecode($url)));
}
function test_redirect()
{
global $CFG;
$feed = new moodle_simplepie();
$feed->set_timeout(self::TIMEOUT);
$feed->set_feed_url(self::REDIRECTURL);
$feed->init();
$this->assertNull($feed->error());
$this->assertEquals($feed->get_title(), 'Moodle News');
$this->assertEquals($feed->get_link(), 'http://moodle.org/mod/forum/view.php?f=1');
}
/**
* cron - goes through all the feeds. If the feed has a skipuntil value
* that is less than the current time cron will attempt to retrieve it
* with the cache duration set to 0 in order to force the retrieval of
* the item and refresh the cache.
*
* If a feed fails then the skipuntil time of that feed is set to be
* later than the next expected cron time. The amount of time will
* increase each time the fetch fails until the maximum is reached.
*
* If a feed that has been failing is successfully retrieved it will
* go back to being handled as though it had never failed.
*
* CRON should therefor process requests for permanently broken RSS
* feeds infrequently, and temporarily unavailable feeds will be tried
* less often until they become available again.
*
* @return boolean Always returns true
*/
function cron()
{
global $CFG, $DB;
require_once $CFG->libdir . '/simplepie/moodle_simplepie.php';
// Get the legacy cron time, strangely the cron property of block_base
// does not seem to get set. This means we must retrive it here.
$this->cron = $DB->get_field('block', 'cron', array('name' => 'rss_client'));
// We are going to measure execution times
$starttime = microtime();
$starttimesec = time();
// Fetch all site feeds.
$rs = $DB->get_recordset('block_rss_client');
$counter = 0;
mtrace('');
foreach ($rs as $rec) {
mtrace(' ' . $rec->url . ' ', '');
// Skip feed if it failed recently.
if ($starttimesec < $rec->skipuntil) {
mtrace('skipping until ' . userdate($rec->skipuntil));
continue;
}
// Fetch the rss feed, using standard simplepie caching
// so feeds will be renewed only if cache has expired
core_php_time_limit::raise(60);
$feed = new moodle_simplepie();
// set timeout for longer than normal to be agressive at
// fetching feeds if possible..
$feed->set_timeout(40);
$feed->set_cache_duration(0);
$feed->set_feed_url($rec->url);
$feed->init();
if ($feed->error()) {
// Skip this feed (for an ever-increasing time if it keeps failing).
$rec->skiptime = $this->calculate_skiptime($rec->skiptime);
$rec->skipuntil = time() + $rec->skiptime;
$DB->update_record('block_rss_client', $rec);
mtrace("Error: could not load/find the RSS feed - skipping for {$rec->skiptime} seconds.");
} else {
mtrace('ok');
// It worked this time, so reset the skiptime.
if ($rec->skiptime > 0) {
$rec->skiptime = 0;
$rec->skipuntil = 0;
$DB->update_record('block_rss_client', $rec);
}
// Only increase the counter when a feed is sucesfully refreshed.
$counter++;
}
}
$rs->close();
// Show times
mtrace($counter . ' feeds refreshed (took ' . microtime_diff($starttime, microtime()) . ' seconds)');
return true;
}
