public function getURLLastModified($url, $symlink, $bucket) { $this->clearstatcache(); if (!$this->fileExists($symlink)) { return array('time' => 0, 'revalidate' => true, 'cache_age' => 0); } $symlinkAbs = $this->absolutize($symlink); $stat = lstat($symlinkAbs); $now = time(); if (readlink($symlinkAbs) == '/dev/null') { $failed_time = $now - $stat['mtime']; if ($failed_time > binarypool_config::getBadUrlExpiry()) { unlink($symlinkAbs); return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time); } $failed_nextfetch = $stat['mtime'] + binarypool_config::getBadUrlExpiry() - $now; throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}"); } $cache_age = $now - $stat['mtime']; $revalidate = false; if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) { $revalidate = true; } return array('time' => filemtime($symlinkAbs), 'revalidate' => $revalidate, 'cache_age' => $cache_age); }
public function getURLLastModified($url, $symlink, $bucket) { $symlink .= '.link'; $now = $this->time; if (!$this->fileExists($symlink)) { return array('time' => 0, 'revalidate' => true, 'cache_age' => 0); } $contents = $this->getFile($symlink); $contents = json_decode($contents, true); if ($contents['link'] == '/dev/null') { // Dead URL $failed_time = $now - $contents['mtime']; if ($failed_time > binarypool_config::getBadUrlExpiry()) { $this->unlink($symlink); return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time); } $failed_nextfetch = $contents['mtime'] + binarypool_config::getBadUrlExpiry() - $now; throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}"); } $cache_age = $now - $contents['mtime']; $revalidate = false; if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) { $revalidate = true; } return array('time' => $contents['mtime'], 'revalidate' => $revalidate, 'cache_age' => $cache_age); }
function testUpdatedSymlink() { $this->assignLastModified(); $cache_age = binarypool_config::getCacheRevalidate('test') + 1; binarypool_views::$lastModified->setReturnValue('lastModified', array('cache_age' => $cache_age)); $asset = $this->createMockAsset(); $storage = $this->createMockStorage($asset); $storage->expectCallCount('symlink', 2); $storage->expectCallCount('relink', 1); $this->assignMockStorageFactory($storage); binarypool_views::created('test', 'foo', array('URL' => 'http://local.ch/foo.gif')); }
protected function getFilesFromUrl() { $url = $this->request->getParam('URL'); $this->log->debug("Downloading file: %s", $url); if (!self::$lastModified) { self::$lastModified = new binarypool_lastmodified(); } $lastmodified = self::$lastModified->lastModified($this->bucket, $url); if (binarypool_config::getCacheRevalidate($this->bucket) === 0) { $lastmodified['time'] = 0; } $tmpfile = tempnam(sys_get_temp_dir(), 'binary'); if ($tmpfile == '' || $tmpfile === FALSE) { throw new binarypool_exception(104, 500, "Could not create temporary file"); } array_push($this->tmpfiles, $tmpfile); $result = array('code' => 0, 'headers' => array(), 'body' => ''); $retries = 3; if ($lastmodified['revalidate']) { $httpc = new binarypool_httpclient(); while ($retries) { try { $result = $httpc->download($url, $tmpfile, $lastmodified['time']); if ($result['code'] < 500) { break; } } catch (binarypool_httpclient_exception $e) { // ignore - dropped connections etc. - retry $this->log->debug("Failed download attempt from %s: %s", $url, $e); } sleep(1); $retries--; } } else { $result['code'] = 304; } if (304 == $result['code']) { $this->log->debug("File %s has not been modified", $url); return array(); } if ($result['code'] != 200 || !filesize($tmpfile)) { binarypool_views::flagBadUrl($this->bucket, $url); throw new binarypool_exception(121, 400, "File could not be fetched from URL: " . $url); } $url_parsed = parse_url($url); $filename = basename($url_parsed['path']); # Restrict filenames TO ALPHANUMS and reduce sequences of '.' to avoid # traversal issues, unicode issues, command injection etc. $filename = preg_replace(array('#\\.{2,}#', '#[^a-zA-Z0-9\\.]+#'), array('.', '_'), $filename); return array('_' => array('file' => $tmpfile, 'filename' => $filename)); }
/** * Create the URL view symlink for files which were downloaded * * @param $asset: An object of binarypool_asset. */ private static function linkURL($bucket, $asset, $metadata) { if (empty($metadata['URL'])) { return; } $assetDir = '../../' . self::getCleanedBasepath($asset); $symlink = self::getDownloadedViewPath($bucket, $metadata['URL']); if (!self::$lastModified) { self::$lastModified = new binarypool_lastmodified(); } $lastmodified = self::$lastModified->lastModified($bucket, $metadata['URL']); if ($lastmodified['cache_age'] > binarypool_config::getCacheRevalidate($bucket)) { self::refreshLink($bucket, $assetDir, $symlink); } else { self::createLink($bucket, $assetDir, $symlink); } }