/**
* Performs synchronisation of an external file if the previous one has expired.
*
* This function must be implemented for external repositories supporting
* FILE_REFERENCE, it is called for existing aliases when their filesize,
* contenthash or timemodified are requested. It is not called for internal
* repositories (see {@link repository::has_moodle_files()}), references to
* internal files are updated immediately when source is modified.
*
* Referenced files may optionally keep their content in Moodle filepool (for
* thumbnail generation or to be able to serve cached copy). In this
* case both contenthash and filesize need to be synchronized. Otherwise repositories
* should use contenthash of empty file and correct filesize in bytes.
*
* Note that this function may be run for EACH file that needs to be synchronised at the
* moment. If anything is being downloaded or requested from external sources there
* should be a small timeout. The synchronisation is performed to update the size of
* the file and/or to update image and re-generated image preview. There is nothing
* fatal if syncronisation fails but it is fatal if syncronisation takes too long
* and hangs the script generating a page.
*
* Note: If you wish to call $file->get_filesize(), $file->get_contenthash() or
* $file->get_timemodified() make sure that recursion does not happen.
*
* Called from {@link stored_file::sync_external_file()}
*
* @uses stored_file::set_missingsource()
* @uses stored_file::set_synchronized()
* @param stored_file $file
* @return bool false when file does not need synchronisation, true if it was synchronised
*/
public function sync_reference(stored_file $file)
{
if ($file->get_repository_id() != $this->id) {
// This should not really happen because the function can be called from stored_file only.
return false;
}
if ($this->has_moodle_files()) {
// References to local files need to be synchronised only once.
// Later they will be synchronised automatically when the source is changed.
if ($file->get_referencelastsync()) {
return false;
}
$fs = get_file_storage();
$params = file_storage::unpack_reference($file->get_reference(), true);
if (!is_array($params) || !($storedfile = $fs->get_file($params['contextid'], $params['component'], $params['filearea'], $params['itemid'], $params['filepath'], $params['filename']))) {
$file->set_missingsource();
} else {
$file->set_synchronized($storedfile->get_contenthash(), $storedfile->get_filesize(), 0, $storedfile->get_timemodified());
}
return true;
}
return false;
}
public function sync_reference(stored_file $file)
{
global $CFG;
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
$ref = unserialize($file->get_reference());
if (!isset($ref->url)) {
// this is an old-style reference in DB. We need to fix it
$ref = unserialize($this->fix_old_style_reference($file->get_reference()));
}
if (!isset($ref->url)) {
return false;
}
$c = new curl();
$url = $this->get_file_download_link($ref->url);
if (file_extension_in_typegroup($ref->path, 'web_image')) {
$saveas = $this->prepare_file('');
try {
$result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($saveas);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} catch (Exception $e) {
}
}
$c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true));
$info = $c->get_info();
if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) {
$filesize = (int) $info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
$file->set_missingsource();
return true;
}
/**
* Downloads the file from external repository and saves it in moodle filepool.
* This function is different from {@link repository::sync_external_file()} because it has
* bigger request timeout and always downloads the content.
*
* This function is invoked when we try to unlink the file from the source and convert
* a reference into a true copy.
*
* @throws exception when file could not be imported
*
* @param stored_file $file
* @param int $maxbytes throw an exception if file size is bigger than $maxbytes (0 means no limit)
*/
public function import_external_file_contents(stored_file $file, $maxbytes = 0)
{
if (!$file->is_external_file()) {
// nothing to import if the file is not a reference
return;
} else {
if ($file->get_repository_id() != $this->id) {
// error
debugging('Repository instance id does not match');
return;
} else {
if ($this->has_moodle_files()) {
// files that are references to local files are already in moodle filepool
// just validate the size
if ($maxbytes > 0 && $file->get_filesize() > $maxbytes) {
throw new file_exception('maxbytes');
}
return;
} else {
if ($maxbytes > 0 && $file->get_filesize() > $maxbytes) {
// note that stored_file::get_filesize() also calls synchronisation
throw new file_exception('maxbytes');
}
$fs = get_file_storage();
$contentexists = $fs->content_exists($file->get_contenthash());
if ($contentexists && $file->get_filesize() && $file->get_contenthash() === sha1('')) {
// even when 'file_storage::content_exists()' returns true this may be an empty
// content for the file that was not actually downloaded
$contentexists = false;
}
$now = time();
if ($file->get_referencelastsync() + $file->get_referencelifetime() >= $now && !$file->get_status() && $contentexists) {
// we already have the content in moodle filepool and it was synchronised recently.
// Repositories may overwrite it if they want to force synchronisation anyway!
return;
} else {
// attempt to get a file
try {
$fileinfo = $this->get_file($file->get_reference());
if (isset($fileinfo['path'])) {
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($fileinfo['path']);
// set this file and other similar aliases synchronised
$lifetime = $this->get_reference_file_lifetime($file->get_reference());
$file->set_synchronized($contenthash, $filesize, 0, $lifetime);
} else {
throw new moodle_exception('errorwhiledownload', 'repository', '', '');
}
} catch (Exception $e) {
if ($contentexists) {
// better something than nothing. We have a copy of file. It's sync time
// has expired but it is still very likely that it is the last version
} else {
throw $e;
}
}
}
}
}
}
}
/**
* Performs synchronisation of an external file if the previous one has expired.
*
* This function must be implemented for external repositories supporting
* FILE_REFERENCE, it is called for existing aliases when their filesize,
* contenthash or timemodified are requested. It is not called for internal
* repositories (see {@link repository::has_moodle_files()}), references to
* internal files are updated immediately when source is modified.
*
* Referenced files may optionally keep their content in Moodle filepool (for
* thumbnail generation or to be able to serve cached copy). In this
* case both contenthash and filesize need to be synchronized. Otherwise repositories
* should use contenthash of empty file and correct filesize in bytes.
*
* Note that this function may be run for EACH file that needs to be synchronised at the
* moment. If anything is being downloaded or requested from external sources there
* should be a small timeout. The synchronisation is performed to update the size of
* the file and/or to update image and re-generated image preview. There is nothing
* fatal if syncronisation fails but it is fatal if syncronisation takes too long
* and hangs the script generating a page.
*
* Note: If you wish to call $file->get_filesize(), $file->get_contenthash() or
* $file->get_timemodified() make sure that recursion does not happen.
*
* Called from {@link stored_file::sync_external_file()}
*
* @uses stored_file::set_missingsource()
* @uses stored_file::set_synchronized()
* @param stored_file $file
* @return bool false when file does not need synchronisation, true if it was synchronised
*/
public function sync_reference(stored_file $file) {
if ($file->get_repository_id() != $this->id) {
// This should not really happen because the function can be called from stored_file only.
return false;
}
if ($this->has_moodle_files()) {
// References to local files need to be synchronised only once.
// Later they will be synchronised automatically when the source is changed.
if ($file->get_referencelastsync()) {
return false;
}
$fs = get_file_storage();
$params = file_storage::unpack_reference($file->get_reference(), true);
if (!is_array($params) || !($storedfile = $fs->get_file($params['contextid'],
$params['component'], $params['filearea'], $params['itemid'], $params['filepath'],
$params['filename']))) {
$file->set_missingsource();
} else {
$file->set_synchronized($storedfile->get_contenthash(), $storedfile->get_filesize());
}
return true;
}
// Backward compatibility (Moodle 2.3-2.5) implementation that calls
// methods repository::get_reference_file_lifetime(), repository::sync_individual_file()
// and repository::get_file_by_reference(). These methods are removed from the
// base repository class but may still be implemented by the child classes.
// THIS IS NOT A GOOD EXAMPLE of implementation. For good examples see the overwriting methods.
if (!method_exists($this, 'get_file_by_reference')) {
// Function get_file_by_reference() is not implemented. No synchronisation.
return false;
}
// Check if the previous sync result is still valid.
if (method_exists($this, 'get_reference_file_lifetime')) {
$lifetime = $this->get_reference_file_lifetime($file->get_reference());
} else {
// Default value that was hardcoded in Moodle 2.3 - 2.5.
$lifetime = 60 * 60 * 24;
}
if (($lastsynced = $file->get_referencelastsync()) && $lastsynced + $lifetime >= time()) {
return false;
}
$cache = cache::make('core', 'repositories');
if (($lastsyncresult = $cache->get('sync:'.$file->get_referencefileid())) !== false) {
if ($lastsyncresult === true) {
// We are in the process of synchronizing this reference.
// Avoid recursion when calling $file->get_filesize() and $file->get_contenthash().
return false;
} else {
// We have synchronised the same reference inside this request already.
// It looks like the object $file was created before the synchronisation and contains old data.
if (!empty($lastsyncresult['missing'])) {
$file->set_missingsource();
} else {
$cache->set('sync:'.$file->get_referencefileid(), true);
if ($file->get_contenthash() != $lastsyncresult['contenthash'] ||
$file->get_filesize() != $lastsyncresult['filesize']) {
$file->set_synchronized($lastsyncresult['contenthash'], $lastsyncresult['filesize']);
}
$cache->set('sync:'.$file->get_referencefileid(), $lastsyncresult);
}
return true;
}
}
// Weird function sync_individual_file() that was present in API in 2.3 - 2.5, default value was true.
if (method_exists($this, 'sync_individual_file') && !$this->sync_individual_file($file)) {
return false;
}
// Set 'true' into the cache to indicate that file is in the process of synchronisation.
$cache->set('sync:'.$file->get_referencefileid(), true);
// Create object with the structure that repository::get_file_by_reference() expects.
$reference = new stdClass();
$reference->id = $file->get_referencefileid();
$reference->reference = $file->get_reference();
$reference->referencehash = sha1($file->get_reference());
$reference->lastsync = $file->get_referencelastsync();
$reference->lifetime = $lifetime;
$fileinfo = $this->get_file_by_reference($reference);
$contenthash = null;
$filesize = null;
$fs = get_file_storage();
if (!empty($fileinfo->filesize)) {
// filesize returned
if (!empty($fileinfo->contenthash) && $fs->content_exists($fileinfo->contenthash)) {
// contenthash is specified and valid
$contenthash = $fileinfo->contenthash;
} else if ($fileinfo->filesize == $file->get_filesize()) {
// we don't know the new contenthash but the filesize did not change,
// assume the contenthash did not change either
$contenthash = $file->get_contenthash();
} else {
// we can't save empty contenthash so generate contenthash from empty string
list($contenthash, $unused1, $unused2) = $fs->add_string_to_pool('');
}
$filesize = $fileinfo->filesize;
} else if (!empty($fileinfo->filepath)) {
// File path returned
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($fileinfo->filepath);
} else if (!empty($fileinfo->handle) && is_resource($fileinfo->handle)) {
// File handle returned
$contents = '';
while (!feof($fileinfo->handle)) {
$contents .= fread($fileinfo->handle, 8192);
}
fclose($fileinfo->handle);
list($contenthash, $filesize, $newfile) = $fs->add_string_to_pool($contents);
} else if (isset($fileinfo->content)) {
// File content returned
list($contenthash, $filesize, $newfile) = $fs->add_string_to_pool($fileinfo->content);
}
if (!isset($contenthash) or !isset($filesize)) {
$file->set_missingsource(null);
$cache->set('sync:'.$file->get_referencefileid(), array('missing' => true));
} else {
// update files table
$file->set_synchronized($contenthash, $filesize);
$cache->set('sync:'.$file->get_referencefileid(),
array('contenthash' => $contenthash, 'filesize' => $filesize));
}
return true;
}
/**
* Synchronize the references.
*
* @param stored_file $file Stored file.
* @return boolean
*/
public function sync_reference(stored_file $file)
{
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Synchronise not more often than once a day.
return false;
}
$c = new curl();
$reference = unserialize(self::convert_to_valid_reference($file->get_reference()));
$url = $reference->downloadurl;
if (file_extension_in_typegroup($file->get_filename(), 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($url, null, array('filepath' => $path, 'timeout' => $CFG->repositorysyncimagetimeout));
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
}
$c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true));
$info = $c->get_info();
if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) {
$filesize = (int) $info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
$file->set_missingsource();
return true;
}
/**
* Repository method to serve the referenced file
*
* @see send_stored_file
*
* @param stored_file $storedfile the file that contains the reference
* @param int $lifetime Number of seconds before the file should expire from caches (null means $CFG->filelifetime)
* @param int $filter 0 (default)=no filtering, 1=all files, 2=html files only
* @param bool $forcedownload If true (default false), forces download of file rather than view in browser/plugin
* @param array $options additional options affecting the file serving
*/
public function send_file($storedfile, $lifetime = null, $filter = 0, $forcedownload = false, array $options = null)
{
$reference = $this->unpack_reference($storedfile->get_reference());
if (false && $_SERVER['SCRIPT_NAME'] !== '/draftfile.php') {
if ($reference['source'] === 'onedrive') {
$sourceclient = $this->get_onedrive_apiclient();
$fileurl = isset($reference['url']) ? $reference['url'] : '';
$embedurl = $sourceclient->get_embed_url($reference['id'], $fileurl);
if (!empty($embedurl)) {
header('Location: ' . $embedurl);
die;
} else {
if (!empty($fileurl)) {
header('Location: ' . $fileurl);
die;
}
}
}
}
try {
$fileinfo = $this->get_file($storedfile->get_reference());
if (isset($fileinfo['path'])) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($fileinfo['path']);
// Set this file and other similar aliases synchronised.
$storedfile->set_synchronized($contenthash, $filesize);
} else {
throw new \moodle_exception('errorwhiledownload', 'repository_office365');
}
if (!is_array($options)) {
$options = [];
}
$options['sendcachedexternalfile'] = true;
send_stored_file($storedfile, $lifetime, $filter, $forcedownload, $options);
} catch (\Exception $e) {
send_file_not_found();
}
}
public function sync_reference(stored_file $file)
{
if ($file->get_referencelastsync() + 60 > time()) {
// Does not cost us much to synchronise within our own filesystem, check every 1 minute.
return false;
}
static $issyncing = false;
if ($issyncing) {
// Avoid infinite recursion when calling $file->get_filesize() and get_contenthash().
return false;
}
$filepath = $this->get_rootpath() . ltrim($file->get_reference(), '/');
if ($this->is_in_repository($file->get_reference()) && file_exists($filepath) && is_readable($filepath)) {
$fs = get_file_storage();
$issyncing = true;
if (file_extension_in_typegroup($filepath, 'web_image')) {
$contenthash = sha1_file($filepath);
if ($file->get_contenthash() == $contenthash) {
// File did not change since the last synchronisation.
$filesize = filesize($filepath);
} else {
// Copy file into moodle filepool (used to generate an image thumbnail).
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($filepath);
}
} else {
// Update only file size so file will NOT be copied into moodle filepool.
$emptyfile = $contenthash = sha1('');
$currentcontenthash = $file->get_contenthash();
if ($currentcontenthash !== $emptyfile && $currentcontenthash === sha1_file($filepath)) {
// File content was synchronised and has not changed since then, leave it.
$contenthash = null;
}
$filesize = filesize($filepath);
}
$issyncing = false;
$file->set_synchronized($contenthash, $filesize);
} else {
$file->set_missingsource();
}
return true;
}
public function sync_reference(stored_file $file)
{
global $USER;
if ($file->get_referencelastsync() + DAYSECS > time() || !$this->connection_result()) {
// Synchronise not more often than once a day.
// if we had several unsuccessfull attempts to connect to server - do not try any more.
return false;
}
$ref = @unserialize(base64_decode($file->get_reference()));
if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) {
// Occurs when the user isn't known..
$file->set_missingsource();
return true;
}
$cookiepathname = $this->prepare_file($USER->id . '_' . uniqid('', true) . '.cookie');
$c = new curl(array('cookie' => $cookiepathname));
if (file_extension_in_typegroup($ref->filename, 'web_image')) {
$path = $this->prepare_file('');
$result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => $CFG->repositorysyncimagetimeout));
if ($result === true) {
$fs = get_file_storage();
list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} else {
$result = $c->head($url, array('followlocation' => true, 'timeout' => $CFG->repositorysyncfiletimeout));
}
// Delete cookie jar.
if (file_exists($cookiepathname)) {
unlink($cookiepathname);
}
$this->connection_result($c->get_errno());
$curlinfo = $c->get_info();
if (isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200 && array_key_exists('download_content_length', $curlinfo) && $curlinfo['download_content_length'] >= 0) {
// we received a correct header and at least can tell the file size
$file->set_synchronized(null, $curlinfo['download_content_length']);
return true;
}
$file->set_missingsource();
return true;
}
/**
* Performs synchronisation of an external file if the previous one has expired.
*
* This function must be implemented for external repositories supporting
* FILE_REFERENCE, it is called for existing aliases when their filesize,
* contenthash or timemodified are requested. It is not called for internal
* repositories (see {@link repository::has_moodle_files()}), references to
* internal files are updated immediately when source is modified.
*
* Referenced files may optionally keep their content in Moodle filepool (for
* thumbnail generation or to be able to serve cached copy). In this
* case both contenthash and filesize need to be synchronized. Otherwise repositories
* should use contenthash of empty file and correct filesize in bytes.
*
* Note that this function may be run for EACH file that needs to be synchronised at the
* moment. If anything is being downloaded or requested from external sources there
* should be a small timeout. The synchronisation is performed to update the size of
* the file and/or to update image and re-generated image preview. There is nothing
* fatal if syncronisation fails but it is fatal if syncronisation takes too long
* and hangs the script generating a page.
*
* Note: If you wish to call $file->get_filesize(), $file->get_contenthash() or
* $file->get_timemodified() make sure that recursion does not happen.
*
* Called from {@link stored_file::sync_external_file()}
*
* @inheritDocs
*/
public function sync_reference(stored_file $file)
{
global $CFG;
if ($file->get_referencelastsync() + DAYSECS > time()) {
// Only synchronise once per day.
return false;
}
$reference = $this->unpack_reference($file->get_reference());
if (!isset($reference->url)) {
// The URL to sync with is missing.
return false;
}
$c = new curl();
$url = $this->get_file_download_link($reference->url);
if (file_extension_in_typegroup($reference->path, 'web_image')) {
$saveas = $this->prepare_file('');
try {
$result = $c->download_one($url, [], ['filepath' => $saveas, 'timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true]);
$info = $c->get_info();
if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) {
$fs = get_file_storage();
list($contenthash, $filesize, ) = $fs->add_file_to_pool($saveas);
$file->set_synchronized($contenthash, $filesize);
return true;
}
} catch (Exception $e) {
// IF the download_one fails, we will attempt to download
// again with get() anyway.
}
}
$c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true));
$info = $c->get_info();
if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) {
$filesize = (int) $info['download_content_length'];
$file->set_synchronized(null, $filesize);
return true;
}
$file->set_missingsource();
return true;
}
