public function sync_reference(stored_file $file) { global $CFG; if ($file->get_referencelastsync() + DAYSECS > time()) { // Synchronise not more often than once a day. return false; } $ref = unserialize($file->get_reference()); if (!isset($ref->url)) { // this is an old-style reference in DB. We need to fix it $ref = unserialize($this->fix_old_style_reference($file->get_reference())); } if (!isset($ref->url)) { return false; } $c = new curl(); $url = $this->get_file_download_link($ref->url); if (file_extension_in_typegroup($ref->path, 'web_image')) { $saveas = $this->prepare_file(''); try { $result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true)); $info = $c->get_info(); if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) { $fs = get_file_storage(); list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($saveas); $file->set_synchronized($contenthash, $filesize); return true; } } catch (Exception $e) { } } $c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true)); $info = $c->get_info(); if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) { $filesize = (int) $info['download_content_length']; $file->set_synchronized(null, $filesize); return true; } $file->set_missingsource(); return true; }
public function test_curl_redirects() { global $CFG; // Test full URL redirects. $testurl = $this->getExternalTestFileUrl('/test_redir.php'); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $response = $curl->getResponse(); $this->assertSame('200 OK', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $response = $curl->getResponse(); $this->assertSame('200 OK', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); // This test was failing for people behind Squid proxies. Squid does not // fully support HTTP 1.1, so converts things to HTTP 1.0, where the name // of the status code is different. reset($response); if (key($response) === 'HTTP/1.0') { $responsecode302 = '302 Moved Temporarily'; } else { $responsecode302 = '302 Found'; } $curl = new curl(); $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $response = $curl->getResponse(); $this->assertSame($responsecode302, reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $response = $curl->getResponse(); $this->assertSame($responsecode302, reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); }
/** * Downloads a file from external repository and saves it in temp dir * * Function get_file() must be implemented by repositories that support returntypes * FILE_INTERNAL or FILE_REFERENCE. It is invoked to pick up the file and copy it * to moodle. This function is not called for moodle repositories, the function * {@link repository::copy_to_area()} is used instead. * * This function can be overridden by subclass if the files.reference field contains * not just URL or if request should be done differently. * * @see curl * @throws file_exception when error occured * * @param string $url the content of files.reference field, in this implementaion * it is asssumed that it contains the string with URL of the file * @param string $filename filename (without path) to save the downloaded file in the * temporary directory, if omitted or file already exists the new filename will be generated * @return array with elements: * path: internal location of the file * url: URL to the source (from parameters) */ public function get_file($url, $filename = '') { global $CFG; $path = $this->prepare_file($filename); $c = new curl(); $result = $c->download_one($url, null, array('filepath' => $path, 'timeout' => $CFG->repositorygetfiletimeout)); if ($result !== true) { throw new moodle_exception('errorwhiledownload', 'repository', '', $result); } return array('path' => $path, 'url' => $url); }
/** * Migrate the references to local files. * * As the APIv1 is reaching its end of life on the 14th of Dec 2013, and we cannot * convert the existing references to new references, we need to convert them * to real files. * * @todo Deprecate/remove this function after the 14th of December 2013. * @return void */ function repository_boxnet_migrate_references_from_apiv1() { global $DB; // A string that the old references contain. $apiv1signature = '/api/1.0/download/'; // Downloading the files could take a very long time! @set_time_limit(0); // Create directory to download temporary files. $dir = make_temp_directory('download/repository_boxnet/'); // Create a dummy file for the broken files. $fs = get_file_storage(); list($dummyhash, $dummysize, $unused) = $fs->add_string_to_pool('Lost reference from Box.net'); // Get the Box.net instances. There should be only one. $sql = "SELECT i.id, i.typeid, r.id, r.type FROM {repository} r, {repository_instances} i WHERE i.typeid = r.id AND r.type = :type"; $ids = $DB->get_fieldset_sql($sql, array('type' => 'boxnet')); if (empty($ids)) { // We did not find any instance of Box.net. Let's just ignore this migration. mtrace('Could not find any instance of the repository, aborting migration...'); return; } // The next bit is copied from the function file_storage::instance_sql_fields() // because it is private and there is nothing in file_storage that suits our needs here. $filefields = array('contenthash', 'pathnamehash', 'contextid', 'component', 'filearea', 'itemid', 'filepath', 'filename', 'userid', 'filesize', 'mimetype', 'status', 'source', 'author', 'license', 'timecreated', 'timemodified', 'sortorder', 'referencefileid'); $referencefields = array('repositoryid' => 'repositoryid', 'reference' => 'reference', 'lifetime' => 'referencelifetime', 'lastsync' => 'referencelastsync'); $fields = array(); $fields[] = 'f.id AS id'; foreach ($filefields as $field) { $fields[] = "f.{$field}"; } foreach ($referencefields as $field => $alias) { $fields[] = "r.{$field} AS {$alias}"; } $fields = implode(', ', $fields); // We are not using repository::convert_references_to_local() or file_storage::get_external_files() // because they would select too many records and load everything in memory as it is not using a recordset. // Also, we filter the results not to get the draft area which should not be converted. list($sqlfragment, $fragmentparams) = $DB->get_in_or_equal($ids, SQL_PARAMS_NAMED); $sql = "SELECT " . $fields . " FROM {files_reference} r LEFT JOIN {files} f ON f.referencefileid = r.id WHERE r.repositoryid $sqlfragment AND f.referencefileid IS NOT NULL AND NOT (f.component = :component AND f.filearea = :filearea)"; // For each reference we download the file. Then we add it to the file pool and update the references. // The reason why we are re-inventing the wheel here is because the current API ends up calling // repository::get_file() which includes a download timeout. As we are trying our best to copy // the files here, we want to ignre any timeout. $filerecords = $DB->get_recordset_sql($sql, array_merge($fragmentparams, array('component' => 'user', 'filearea' => 'draft'))); $referenceids = array(); foreach ($filerecords as $filerecord) { $file = $fs->get_file_instance($filerecord); $reference = unserialize(repository_boxnet::convert_to_valid_reference($file->get_reference())); if (empty($reference->downloadurl)) { // Something is wrong... mtrace('Skipping malformed reference (id: ' . $file->get_referencefileid() . ')'); continue; } else if (strpos($reference->downloadurl, $apiv1signature) === false) { // This is not an old reference, we are not supposed to work on thos. mtrace('Skipping non APIv1 reference (id: ' . $file->get_referencefileid() . ')'); continue; } else if (isset($referenceids[$file->get_referencefileid()])) { // We have already worked on that reference, we skip any other file related to it. // We cannot work on them here because they have been updated in the database but our // recordset does not have those new values. They will be taken care of after this foreach. continue; } mtrace('Starting migration of file reference ' . $file->get_referencefileid()); // Manually import the file to the file pool to prevent timeout limitations of the repository method get_file(). // We ignore the fact that the content of the file could exist locally because we want to synchronize the file // now to prevent the repository to try to download the file as well. $saveas = $dir . uniqid('', true) . '_' . time() . '.tmp'; $c = new curl(); $result = $c->download_one($reference->downloadurl, null, array('filepath' => $saveas, 'followlocation' => true)); $info = $c->get_info(); if ($result !== true || !isset($info['http_code']) || $info['http_code'] != 200) { // There was a problem while trying to download the reference... if ($fs->content_exists($file->get_contenthash()) && $file->get_contenthash() != sha1('')) { // Fortunately we already had a local version of this reference, so we keep it. We have to // set it synchronized or there is a risk that repository::sync_reference() will try to download // the file again. We cannot use $file->get_contenthash() and $file->get_filesize() because they // cause repository::sync_reference() to be called. $file->set_synchronized($filerecord->contenthash, $filerecord->filesize, 0, DAYSECS); mtrace('Could not download reference, using last synced file. (id: ' . $file->get_referencefileid() . ')'); } else { // We don't know what the file was, but what can we do? In order to prevent a re-attempt to fetch the // file in the next bit of this script (import_external_file()), we set a dummy content to the reference. $file->set_synchronized($dummyhash, $dummysize, 0, DAYSECS); mtrace('Could not download reference, dummy file used. (id: ' . $file->get_referencefileid() . ')'); } } else { try { // The file has been downloaded, we add it to the file pool and synchronize // all the files using this reference. list($contenthash, $filesize, $unused) = $fs->add_file_to_pool($saveas); $file->set_synchronized($contenthash, $filesize, 0, DAYSECS); } catch (moodle_exception $e) { // Something wrong happened... mtrace('Something went wrong during sync (id: ' . $file->get_referencefileid() . ')'); } } // Log the reference IDs. $referenceids[$file->get_referencefileid()] = $file->get_referencefileid(); // Now that the file is downloaded, we can loop over all the files using this reference // to convert them to local copies. We have chosen to do that in this loop so that if the // execution fails in the middle, we would not have to redownload the files again and again. // By the way, we cannot use the records fetched in $filerecords because they will not be updated. $sql = "SELECT " . $fields . " FROM {files} f LEFT JOIN {files_reference} r ON f.referencefileid = r.id WHERE f.referencefileid = :refid AND NOT (f.component = :component AND f.filearea = :filearea)"; $reffilerecords = $DB->get_recordset_sql($sql, array('component' => 'user', 'filearea' => 'draft', 'refid' => $file->get_referencefileid())); foreach ($reffilerecords as $reffilerecord) { $reffile = $fs->get_file_instance($reffilerecord); try { // Updating source to remove trace of APIv1 URL. $reffile->set_source('Box APIv1 reference'); } catch (moodle_exception $e) { // Do not fail for this lame reason... } try { $fs->import_external_file($reffile); mtrace('File using reference converted to local file (id: ' . $reffile->get_id() . ')'); } catch (moodle_exception $e) { // Oh well... we tried what we could! $reffile->delete_reference(); mtrace('Failed to convert file from reference to local file, sorry! (id: ' . $reffile->get_id() . ')'); } } } mtrace('Migration finished.'); }
$image_height = optional_param("height", 128, PARAM_INT); // optional param for forcing image reload $force_reload = optional_param("force", false, PARAM_BOOL); // get a reference to the cache $cache = cache::make('repository_omero', 'thumbnail_cache'); // computer the key of the cached element $cache_key = urlencode("{$omero_server}-{$image_id}-{$image_last_update}"); // try to the get thumbnail from the cache $file = $force_reload ? null : $cache->get($cache_key); // download the file is needed and update the cache if ($force_reload || !$file) { $cache->acquire_lock($cache_key); try { $file = $force_reload ? null : $cache->get($cache_key); if (!$file) { $url = "{$omero_server}/ome_seadragon/deepzoom/get/thumbnail/{$image_id}.dzi"; //$url = "${omero_server}/webgateway/render_thumbnail/${image_id}/${image_width}/${image_height}"; $c = new curl(); $file = $c->download_one($url, array("size" => $image_height, "width" => $image_width, "height" => $image_height)); if ($file) { $cache->set($cache_key, $file); } } } finally { $cache->release_lock($cache_key); } } // send the thumbnail header("Content-Type: image/png"); echo $file; exit;
/** * Returns information about file in this repository by reference * {@link repository::get_file_reference()} * {@link repository::get_file()} * * Returns null if file not found or is not readable * * @param stdClass $reference file reference db record * @return null|stdClass with attribute 'filepath' */ public function get_file_by_reference($reference) { $array = explode('/', $reference->reference); $fileid = array_pop($array); $fileinfo = $this->boxclient->get_file_info($fileid, self::SYNCFILE_TIMEOUT); if ($fileinfo) { $size = (int)$fileinfo->size; if (file_extension_in_typegroup($fileinfo->file_name, 'web_image')) { // this is an image - download it to moodle $path = $this->prepare_file(''); $c = new curl; $result = $c->download_one($reference->reference, null, array('filepath' => $path, 'timeout' => self::SYNCIMAGE_TIMEOUT)); if ($result === true) { return (object)array('filepath' => $path); } } return (object)array('filesize' => $size); } return null; }
/** * Downloads a file from external repository and saves it in temp dir * * Function get_file() must be implemented by repositories that support returntypes * FILE_INTERNAL or FILE_REFERENCE. It is invoked to pick up the file and copy it * to moodle. This function is not called for moodle repositories, the function * {@link repository::copy_to_area()} is used instead. * * This function can be overridden by subclass if the files.reference field contains * not just URL or if request should be done differently. * * @see curl * @throws file_exception when error occured * * @param string $url the content of files.reference field, in this implementaion * it is asssumed that it contains the string with URL of the file * @param string $filename filename (without path) to save the downloaded file in the * temporary directory, if omitted or file already exists the new filename will be generated * @return array with elements: * path: internal location of the file * url: URL to the source (from parameters) */ public function get_file($url, $filename = '') { $path = $this->prepare_file($filename); $c = new curl; $result = $c->download_one($url, null, array('filepath' => $path, 'timeout' => self::GETFILE_TIMEOUT)); if ($result !== true) { throw new moodle_exception('errorwhiledownload', 'repository', '', $result); } return array('path'=>$path, 'url'=>$url); }
/** * Returns information about file in this repository by reference * {@link repository::get_file_reference()} * {@link repository::get_file()} * * Returns null if file not found or is not readable * * @param stdClass $reference file reference db record * @return null|stdClass that has 'filepath' property */ public function get_file_by_reference($reference) { global $USER; $ref = unserialize($reference->reference); if (!isset($ref->url)) { // this is an old-style reference in DB. We need to fix it $ref = unserialize($this->fix_old_style_reference($reference->reference)); } if (!isset($ref->url)) { return null; } $c = new curl; $url = $this->get_file_download_link($ref->url); if (file_extension_in_typegroup($ref->path, 'web_image')) { $saveas = $this->prepare_file(''); try { $result = $c->download_one($url, array(), array('filepath' => $saveas, 'timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true)); $info = $c->get_info(); if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) { return (object)array('filepath' => $saveas); } } catch (Exception $e) {} } $c->get($url, null, array('timeout' => self::SYNCIMAGE_TIMEOUT, 'followlocation' => true, 'nobody' => true)); $info = $c->get_info(); if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) { return (object)array('filesize' => (int)$info['download_content_length']); } return null; }
/** * Synchronize the references. * * @param stored_file $file Stored file. * @return boolean */ public function sync_reference(stored_file $file) { if ($file->get_referencelastsync() + DAYSECS > time()) { // Synchronise not more often than once a day. return false; } $c = new curl(); $reference = unserialize(self::convert_to_valid_reference($file->get_reference())); $url = $reference->downloadurl; if (file_extension_in_typegroup($file->get_filename(), 'web_image')) { $path = $this->prepare_file(''); $result = $c->download_one($url, null, array('filepath' => $path, 'timeout' => $CFG->repositorysyncimagetimeout)); $info = $c->get_info(); if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) { $fs = get_file_storage(); list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path); $file->set_synchronized($contenthash, $filesize); return true; } } $c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true)); $info = $c->get_info(); if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) { $filesize = (int) $info['download_content_length']; $file->set_synchronized(null, $filesize); return true; } $file->set_missingsource(); return true; }
/** * Download the given file into the given destination. * * This is basically a simplified version of {@link download_file_content()} from * Moodle itself, tuned for fetching files from moodle.org servers. Same code is used * in mdeploy.php for fetching available updates. * * @param string $source file url starting with http(s):// * @param string $target store the downloaded content to this file (full path) * @throws tool_installaddon_installer_exception */ public function download_file($source, $target) { global $CFG; require_once $CFG->libdir . '/filelib.php'; $targetfile = fopen($target, 'w'); if (!$targetfile) { throw new tool_installaddon_installer_exception('err_download_write_file', $target); } $options = array('file' => $targetfile, 'timeout' => 300, 'followlocation' => true, 'maxredirs' => 3, 'ssl_verifypeer' => true, 'ssl_verifyhost' => 2); $curl = new curl(array('proxy' => true)); $result = $curl->download_one($source, null, $options); $curlinfo = $curl->get_info(); fclose($targetfile); if ($result !== true) { throw new tool_installaddon_installer_exception('err_curl_exec', array('url' => $source, 'errorno' => $curl->get_errno(), 'error' => $result)); } else { if (empty($curlinfo['http_code']) or $curlinfo['http_code'] != 200) { throw new tool_installaddon_installer_exception('err_curl_http_code', array('url' => $source, 'http_code' => $curlinfo['http_code'])); } else { if (isset($curlinfo['ssl_verify_result']) and $curlinfo['ssl_verify_result'] != 0) { throw new tool_installaddon_installer_exception('err_curl_ssl_verify', array('url' => $source, 'ssl_verify_result' => $curlinfo['ssl_verify_result'])); } } } }
/** * Returns information about file in this repository by reference * * If the file is an image we download the contents and save it in our filesystem * so we can generate thumbnails. Otherwise we just request the file size. * Returns null if file not found or can not be accessed * * @param stdClass $reference file reference db record * @return stdClass|null contains one of the following: * - 'filesize' (for non-image files or files we failed to retrieve fully because of timeout) * - 'filepath' (for image files that we retrieived and saved) */ public function get_file_by_reference($reference) { global $USER; $ref = @unserialize(base64_decode($reference->reference)); if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) { // Occurs when the user isn't known.. return null; } $return = null; $cookiepathname = $this->prepare_file($USER->id. '_'. uniqid('', true). '.cookie'); $c = new curl(array('cookie' => $cookiepathname)); if (file_extension_in_typegroup($ref->filename, 'web_image')) { $path = $this->prepare_file(''); $result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => self::SYNCIMAGE_TIMEOUT)); if ($result === true) { $return = (object)array('filepath' => $path); } } else { $result = $c->head($url, array('followlocation' => true, 'timeout' => self::SYNCFILE_TIMEOUT)); } // Delete cookie jar. if (file_exists($cookiepathname)) { unlink($cookiepathname); } $this->connection_result($c->get_errno()); $curlinfo = $c->get_info(); if ($return === null && isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200 && array_key_exists('download_content_length', $curlinfo) && $curlinfo['download_content_length'] >= 0) { // we received a correct header and at least can tell the file size $return = (object)array('filesize' => $curlinfo['download_content_length']); } return $return; }
public function sync_reference(stored_file $file) { global $USER; if ($file->get_referencelastsync() + DAYSECS > time() || !$this->connection_result()) { // Synchronise not more often than once a day. // if we had several unsuccessfull attempts to connect to server - do not try any more. return false; } $ref = @unserialize(base64_decode($file->get_reference())); if (!isset($ref->url) || !($url = $this->appendtoken($ref->url))) { // Occurs when the user isn't known.. $file->set_missingsource(); return true; } $cookiepathname = $this->prepare_file($USER->id . '_' . uniqid('', true) . '.cookie'); $c = new curl(array('cookie' => $cookiepathname)); if (file_extension_in_typegroup($ref->filename, 'web_image')) { $path = $this->prepare_file(''); $result = $c->download_one($url, null, array('filepath' => $path, 'followlocation' => true, 'timeout' => $CFG->repositorysyncimagetimeout)); if ($result === true) { $fs = get_file_storage(); list($contenthash, $filesize, $newfile) = $fs->add_file_to_pool($path); $file->set_synchronized($contenthash, $filesize); return true; } } else { $result = $c->head($url, array('followlocation' => true, 'timeout' => $CFG->repositorysyncfiletimeout)); } // Delete cookie jar. if (file_exists($cookiepathname)) { unlink($cookiepathname); } $this->connection_result($c->get_errno()); $curlinfo = $c->get_info(); if (isset($curlinfo['http_code']) && $curlinfo['http_code'] == 200 && array_key_exists('download_content_length', $curlinfo) && $curlinfo['download_content_length'] >= 0) { // we received a correct header and at least can tell the file size $file->set_synchronized(null, $curlinfo['download_content_length']); return true; } $file->set_missingsource(); return true; }
/** * Test curl class. */ public function test_curl_class() { global $CFG; // Test https success. $testhtml = $this->getExternalTestFileUrl('/test.html'); $curl = new curl(); $contents = $curl->get($testhtml); $this->assertSame('47250a973d1b88d9445f94db4ef2c97a', md5($contents)); $this->assertSame(0, $curl->get_errno()); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get($testhtml, array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame($contents, file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one($testhtml, array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame($contents, file_get_contents($tofile)); @unlink($tofile); // Test full URL redirects. $testurl = $this->getExternalTestFileUrl('/test_redir.php'); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); // Test relative location redirects. $testurl = $this->getExternalTestFileUrl('/test_relative_redir.php'); $curl = new curl(); $contents = $curl->get($testurl); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get($testurl); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); // Test different redirect types. $testurl = $this->getExternalTestFileUrl('/test_relative_redir.php'); $curl = new curl(); $contents = $curl->get("{$testurl}?type=301"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?type=301"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?type=302"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?type=302"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?type=303"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?type=303"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?type=307"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?type=307"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?type=308"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?type=308"); $this->assertSame(0, $curl->get_errno()); $this->assertSame(1, $curl->info['redirect_count']); $this->assertSame('done', $contents); }
public function test_curl_redirects() { global $CFG; // Test full URL redirects. $testurl = $this->getExternalTestFileUrl('/test_redir.php'); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $response = $curl->getResponse(); $this->assertSame('200 OK', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 2)); $response = $curl->getResponse(); $this->assertSame('200 OK', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(2, $curl->info['redirect_count']); $this->assertSame('done', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $response = $curl->getResponse(); $this->assertSame('302 Found', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=3", array(), array('CURLOPT_FOLLOWLOCATION' => 0)); $response = $curl->getResponse(); $this->assertSame('302 Found', reset($response)); $this->assertSame(0, $curl->get_errno()); $this->assertSame(302, $curl->info['http_code']); $this->assertSame('', $contents); $curl = new curl(); $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $curl->emulateredirects = true; $contents = $curl->get("{$testurl}?redir=2", array(), array('CURLOPT_MAXREDIRS' => 1)); $this->assertSame(CURLE_TOO_MANY_REDIRECTS, $curl->get_errno()); $this->assertNotEmpty($contents); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $fp = fopen($tofile, 'w'); $result = $curl->get("{$testurl}?redir=1", array(), array('CURLOPT_FILE' => $fp)); $this->assertTrue($result); fclose($fp); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); $curl = new curl(); $curl->emulateredirects = true; $tofile = "{$CFG->tempdir}/test.html"; @unlink($tofile); $result = $curl->download_one("{$testurl}?redir=1", array(), array('filepath' => $tofile)); $this->assertTrue($result); $this->assertFileExists($tofile); $this->assertSame('done', file_get_contents($tofile)); @unlink($tofile); }
/** * Downloads a repository file and saves to a path. * * @param string $ref reference to the file * @param string $filename to save file as * @return array */ public function get_file($ref, $filename = '') { $ref = unserialize(self::convert_to_valid_reference($ref)); $path = $this->prepare_file($filename); if (!empty($ref->downloadurl)) { $c = new curl(); $result = $c->download_one($ref->downloadurl, null, array('filepath' => $filename, 'timeout' => self::GETFILE_TIMEOUT, 'followlocation' => true)); $info = $c->get_info(); if ($result !== true || !isset($info['http_code']) || $info['http_code'] != 200) { throw new moodle_exception('errorwhiledownload', 'repository', '', $result); } } else { if (!$this->boxnetclient->download_file($ref->fileid, $path)) { throw new moodle_exception('cannotdownload', 'repository'); } } return array('path' => $path); }
/** * Performs synchronisation of an external file if the previous one has expired. * * This function must be implemented for external repositories supporting * FILE_REFERENCE, it is called for existing aliases when their filesize, * contenthash or timemodified are requested. It is not called for internal * repositories (see {@link repository::has_moodle_files()}), references to * internal files are updated immediately when source is modified. * * Referenced files may optionally keep their content in Moodle filepool (for * thumbnail generation or to be able to serve cached copy). In this * case both contenthash and filesize need to be synchronized. Otherwise repositories * should use contenthash of empty file and correct filesize in bytes. * * Note that this function may be run for EACH file that needs to be synchronised at the * moment. If anything is being downloaded or requested from external sources there * should be a small timeout. The synchronisation is performed to update the size of * the file and/or to update image and re-generated image preview. There is nothing * fatal if syncronisation fails but it is fatal if syncronisation takes too long * and hangs the script generating a page. * * Note: If you wish to call $file->get_filesize(), $file->get_contenthash() or * $file->get_timemodified() make sure that recursion does not happen. * * Called from {@link stored_file::sync_external_file()} * * @inheritDocs */ public function sync_reference(stored_file $file) { global $CFG; if ($file->get_referencelastsync() + DAYSECS > time()) { // Only synchronise once per day. return false; } $reference = $this->unpack_reference($file->get_reference()); if (!isset($reference->url)) { // The URL to sync with is missing. return false; } $c = new curl(); $url = $this->get_file_download_link($reference->url); if (file_extension_in_typegroup($reference->path, 'web_image')) { $saveas = $this->prepare_file(''); try { $result = $c->download_one($url, [], ['filepath' => $saveas, 'timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true]); $info = $c->get_info(); if ($result === true && isset($info['http_code']) && $info['http_code'] == 200) { $fs = get_file_storage(); list($contenthash, $filesize, ) = $fs->add_file_to_pool($saveas); $file->set_synchronized($contenthash, $filesize); return true; } } catch (Exception $e) { // IF the download_one fails, we will attempt to download // again with get() anyway. } } $c->get($url, null, array('timeout' => $CFG->repositorysyncimagetimeout, 'followlocation' => true, 'nobody' => true)); $info = $c->get_info(); if (isset($info['http_code']) && $info['http_code'] == 200 && array_key_exists('download_content_length', $info) && $info['download_content_length'] >= 0) { $filesize = (int) $info['download_content_length']; $file->set_synchronized(null, $filesize); return true; } $file->set_missingsource(); return true; }